diff --git a/Documentation/devicetree/bindings/display/apple,dcp.yaml b/Documentation/devicetree/bindings/display/apple,dcp.yaml index b45a737748180..6a494d59a7f89 100644 --- a/Documentation/devicetree/bindings/display/apple,dcp.yaml +++ b/Documentation/devicetree/bindings/display/apple,dcp.yaml @@ -97,13 +97,32 @@ properties: clocks: minItems: 1 - maxItems: 2 + maxItems: 64 clock-names: minItems: 1 - items: - - const: clock-154 - - const: clock-194 + maxItems: 64 + + apple,clock-ids: + oneOf: + - type: boolean + - $ref: /schemas/types.yaml#/definitions/uint32-array + minItems: 1 + maxItems: 64 + description: | + Ordered firmware provider clock identifiers. Callback clock indices + select the corresponding entry in clocks. An explicitly empty table + is allowed when the provider has no clocks. + + apple,firmware-physical-regs: + $ref: /schemas/types.yaml#/definitions/uint64-array + minItems: 1 + maxItems: 10 + description: | + Coprocessor physical base addresses in display register-table order, + including the bandwidth scratch and optional doorbell windows. + The number of entries must match the complete register table. + Omit on identity-mapped systems; reg supplies the physical view. memory-region: minItems: 1 @@ -446,12 +465,39 @@ allOf: then: required: - apple,bw-scratch + - if: + properties: + compatible: + contains: + enum: ['apple,t6050-dcp', 'apple,t6050-dcpext'] + then: + required: + - apple,clock-ids + - memory-region + - memory-region-names + - apple,firmware-compat + properties: + apple,firmware-compat: + items: + - const: 27 + - const: 0 + - const: 0 + else: + properties: + clocks: + maxItems: 2 + clock-names: + items: + - const: clock-154 + - const: clock-194 + apple,clock-ids: false + apple,firmware-physical-regs: false - if: not: properties: compatible: contains: - enum: ['apple,t8140-dcp', 'apple,t8140-dcpext'] + enum: ['apple,t8140-dcp', 'apple,t8140-dcpext', 'apple,t6050-dcp', 'apple,t6050-dcpext'] then: properties: apple,power-lut-data-x: false @@ -463,7 +509,8 @@ allOf: properties: compatible: contains: - enum: ['apple,t8140-dcp', 'apple,t8140-dcpext'] + enum: ['apple,t8140-dcp', 'apple,t8140-dcpext', + 'apple,t6050-dcp', 'apple,t6050-dcpext'] then: properties: resets: false diff --git a/Documentation/devicetree/bindings/gpu/apple,agx.yaml b/Documentation/devicetree/bindings/gpu/apple,agx.yaml index 202544c07b0bb..992783b2dc68f 100644 --- a/Documentation/devicetree/bindings/gpu/apple,agx.yaml +++ b/Documentation/devicetree/bindings/gpu/apple,agx.yaml @@ -17,6 +17,7 @@ properties: - apple,agx-g13s - apple,agx-g14g - apple,agx-g14s + - apple,agx-g17s - apple,agx-t6040 - apple,agx-t6041 - apple,agx-t6050 @@ -40,9 +41,11 @@ properties: - const: apple,agx-g14s reg: + minItems: 1 maxItems: 2 reg-names: + minItems: 1 maxItems: 2 power-domains: @@ -200,6 +203,14 @@ properties: AFR leakage. Per-chip efuse values decoded by the boot loader, stored as IEEE-754 binary32 bit patterns in big-endian cells. + apple,leakage-calibration-words: + $ref: /schemas/types.yaml#/definitions/uint32-array + minItems: 3 + maxItems: 3 + description: + Raw per-chip GPU leakage calibration words for the M5 Pro controller + defaults. Supplied by the boot loader, never fixed in a board DT. + apple,firmware-abi: $ref: /schemas/types.yaml#/definitions/uint32-array minItems: 3 @@ -213,6 +224,46 @@ required: - reg allOf: + - if: + properties: + compatible: + contains: + const: apple,agx-g17s + then: + description: + The M5 Pro GPU uses one MMIO range and two firmware mailbox roles. + Its firmware graph supplies the performance tables, so it has no + operating-point table in the board DT. + properties: + reg: + maxItems: 1 + reg-names: + items: + - const: sgx + mboxes: + items: + - description: Primary GFX firmware mailbox + - description: Secondary GFX firmware mailbox + memory-region: + items: + - description: Region containing GPU MMU TTBs + - description: Region containing GPU MMU page tables + - description: Shared handoff structure for VM coordination + - description: Data segment of the loaded GPU firmware + - description: Firmware kernel-window page tables + - description: Text segment identifying the loaded GPU firmware + memory-region-names: + items: + - const: ttbs + - const: pagetables + - const: handoff + - const: fw-data + - const: l2-pagetables + - const: fw-text + required: + - power-domains + - reg-names + - memory-region-names - if: properties: compatible: @@ -409,6 +460,7 @@ allOf: - apple,agx-t6041 - apple,agx-t8132 - apple,agx-t8140 + - apple,agx-g17s then: properties: reg: diff --git a/Documentation/hwmon/macsmc-hwmon.rst b/Documentation/hwmon/macsmc-hwmon.rst index 23baece30154a..46a6d504b7f5c 100644 --- a/Documentation/hwmon/macsmc-hwmon.rst +++ b/Documentation/hwmon/macsmc-hwmon.rst @@ -36,6 +36,13 @@ The driver leaves fan policy to the SMC until userspace writes a nonzero shutdown and removal release only fans taken over by such a request; resume does not reapply it. A new target write is required after resume. +A kernel thermal policy (the M5 GPU governor) sets the fans through +macsmc_hwmon_fans_request(), as a share of each fan's range, on every fan +whose node names its mode, target, minimum and maximum keys, with or without +apple,fan-manual-control. It uses the same manual mode, leaves fans userspace +holds alone, and its fans return to the SMC at suspend, shutdown and removal +like userspace's. + SoC die temperatures are described by the ``apple,soc-die-keys`` string list on the hwmon device node. The driver appends these temperature channels after the ``temperature-*`` children, in list order, with labels ``SoC die ``. diff --git a/arch/arm64/boot/dts/apple/t6050-j714s-gpu-panel.dtsi b/arch/arm64/boot/dts/apple/t6050-j714s-gpu-panel.dtsi new file mode 100644 index 0000000000000..de9e9ba09285d --- /dev/null +++ b/arch/arm64/boot/dts/apple/t6050-j714s-gpu-panel.dtsi @@ -0,0 +1,206 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * J714s internal panel, external display engines and GPU (G17S), for the boot + * loader's display and GPU handoff. Everything here stays disabled: the loader + * checks the ADT and the live DCP, DART and GPU firmware state, publishes the + * DCP's firmware properties and fills the GPU reservations, and only then + * enables the nodes, the GPU last. A loader without the handoff, or one whose + * checks fail, leaves the boot framebuffer to simpledrm and the GPU off. + * + * Values are from the J714sAP 27.0 ADT (dcp, dart-dcp, dart-disp0, dcpext*, + * dart-dcpext*, dart-dispext*, sgx, gfx-asc, gfx1-asc, pmgr) and the GPU + * firmware's own boot arguments (dArW, the two ASC wrappers). + */ + +/ { + aliases { + /* The loader writes apple,notch-height into the "dcp" alias */ + dcp = &dcp; + disp0 = &display; + /* The loader finds the GPU through this alias */ + gpu = &sgx; + }; + + /* As on t6050-ultra.dtsi, with the internal display's DART (disp0, SID 0) */ + display: display-subsystem { + compatible = "apple,display-subsystem"; + iommus = <&disp0_dart 0>; + status = "disabled"; + }; + + reserved-memory { + /* Filled and enabled by the loader from the ADT */ + gpu_fw_text: gpu-fw-text { + reg = <0 0 0 0>; + no-map; + status = "disabled"; + }; + + gpu_fw_data: gpu-fw-data { + reg = <0 0 0 0>; + no-map; + status = "disabled"; + }; + + uat_ttbs: uat-ttbs { + reg = <0 0 0 0>; + no-map; + status = "disabled"; + }; + + uat_pagetables: uat-pagetables { + reg = <0 0 0 0>; + no-map; + status = "disabled"; + }; + + uat_l2_pagetables: uat-l2-pagetables { + reg = <0 0 0 0>; + no-map; + status = "disabled"; + }; + + uat_handoff: uat-handoff { + reg = <0 0 0 0>; + no-map; + status = "disabled"; + }; + }; +}; + +/* ADT dart-dcp and dart-disp0: vm-base 1 TiB, vm-size 3 TiB */ +&dcp_dart { + apple,dma-range = <0x100 0x0 0x300 0x0>; +}; + +&disp0_dart { + apple,dma-range = <0x100 0x0 0x300 0x0>; +}; + +&dcp { + /* + * The 14.2-inch 3024x1964 mini-LED panel of J314/J414/J514. The OS takes + * the dimensions from the ADT when it has them; this J714s ADT does not. + */ + panel { + compatible = "apple,panel-j714", "apple,panel-mini-led", "apple,panel"; + width-mm = <302>; + height-mm = <196>; + adj-height-mm = <189>; + }; +}; + +/* + * The four external display engines (dcpext0..3): the loader hands each over + * when its checks pass, else leaves it disabled. No display is described on + * them: dcpext0 loses its Type-C route, whose crossbar J714s keeps disabled. + */ +&dcpext0 { + /delete-property/ phys; + /delete-property/ phy-names; + /delete-property/ mux-controls; + /delete-property/ mux-control-names; + /delete-property/ apple,typec-dptx-phys; + /delete-property/ apple,typec-mux-indices; + /delete-property/ apple,connector-type; + /delete-node/ typec-routes; +}; + +/* The ADT lists no clocks for the external engines (clock-ids = []) */ +&dcpext1 { + apple,clock-ids; +}; + +&dcpext2 { + apple,clock-ids; +}; + +&dcpext3 { + apple,clock-ids; +}; + +/* ADT dart-dcpextN and dart-dispextN: vm-base 1 TiB, vm-size 3 TiB, as dcpext0's */ +&dcpext1_dart { + apple,dma-range = <0x100 0x0 0x300 0x0>; +}; + +&dispext1_dart { + apple,dma-range = <0x100 0x0 0x300 0x0>; +}; + +&dcpext2_dart { + apple,dma-range = <0x100 0x0 0x300 0x0>; +}; + +&dispext2_dart { + apple,dma-range = <0x100 0x0 0x300 0x0>; +}; + +&dcpext3_dart { + apple,dma-range = <0x100 0x0 0x300 0x0>; +}; + +&dispext3_dart { + apple,dma-range = <0x100 0x0 0x300 0x0>; +}; + +/* The inventory ASC nodes; the mailboxes below describe the same windows. */ +/delete-node/ &gfx_asc; +/delete-node/ &gfx1_asc; + +&{/soc} { + /* + * ADT gfx-asc and gfx1-asc at their physical windows: the wrapper, the + * 8-byte iop-vbar, and their second and third interrupts (send-empty, + * recv-not-empty), as T8140/T8142. + */ + gfx_mbox: mailbox@2302600000 { + compatible = "apple,t6050-j775d-ascwrap-v6"; + reg = <0x23 0x2600000 0x0 0x88000>, + <0x23 0x2050000 0x0 0x8>; + reg-names = "wrapper", "iop-vbar"; + interrupts = , + ; + interrupt-names = "send-empty", "recv-not-empty"; + apple,firmware-role = "GFX"; + power-domains = <&ps_gfx>; + #mbox-cells = <0>; + status = "disabled"; + }; + + gfx1_mbox: mailbox@2302e00000 { + compatible = "apple,t6050-j775d-ascwrap-v6"; + reg = <0x23 0x2e00000 0x0 0x88000>, + <0x23 0x2850000 0x0 0x8>; + reg-names = "wrapper", "iop-vbar"; + interrupts = , + ; + interrupt-names = "send-empty", "recv-not-empty"; + apple,firmware-role = "GFX1"; + power-domains = <&ps_gfx>; + #mbox-cells = <0>; + status = "disabled"; + }; +}; + +/* The ADT marks GPX and AFR, GFX's parents, critical. */ +&ps_gpx { + apple,always-on; +}; + +&ps_afr { + apple,always-on; +}; + +&sgx { + compatible = "apple,agx-g17s"; + /* ADT sgx reg[0]; the driver maps the first 16 MiB as "sgx" */ + reg = <0x23 0x0 0x0 0x3fdc000>; + reg-names = "sgx"; + /delete-property/ interrupts; + mboxes = <&gfx_mbox>, <&gfx1_mbox>; + memory-region = <&uat_ttbs>, <&uat_pagetables>, <&uat_handoff>, + <&gpu_fw_data>, <&uat_l2_pagetables>, <&gpu_fw_text>; + memory-region-names = "ttbs", "pagetables", "handoff", "fw-data", + "l2-pagetables", "fw-text"; +}; diff --git a/arch/arm64/boot/dts/apple/t6050-j714s.dts b/arch/arm64/boot/dts/apple/t6050-j714s.dts index 79544fbdc4c16..301a1e02ec041 100644 --- a/arch/arm64/boot/dts/apple/t6050-j714s.dts +++ b/arch/arm64/boot/dts/apple/t6050-j714s.dts @@ -500,6 +500,9 @@ /* The same T6050 camera as J716c (J714sAP ADT); the loader sets the J714s sensor type. */ #include "t6050-j716c-isp.dtsi" +/* The GPU and the internal panel, for the boot loader's handoff */ +#include "t6050-j714s-gpu-panel.dtsi" + /* * Audio is not brought up: ADT i2c1 has the left sn012776 amplifiers and i2c2 the * cs42l84 codec, so both buses stay off. diff --git a/drivers/gpu/drm/apple/Kconfig b/drivers/gpu/drm/apple/Kconfig index 7575fb03695ec..3f09ef36c656d 100644 --- a/drivers/gpu/drm/apple/Kconfig +++ b/drivers/gpu/drm/apple/Kconfig @@ -6,6 +6,8 @@ config DRM_APPLE depends on APPLE_RTKIT depends on TYPEC || !TYPEC depends on OF_ADDRESS + # The firmware-27 remote buffer path calls into the DART driver. + depends on APPLE_DART || !APPLE_DART select APPLE_PMP_REPORT select OF_DYNAMIC select DRM_CLIENT_SELECTION diff --git a/drivers/gpu/drm/apple/Makefile b/drivers/gpu/drm/apple/Makefile index 964da993e8f28..ccdb10a97bec3 100644 --- a/drivers/gpu/drm/apple/Makefile +++ b/drivers/gpu/drm/apple/Makefile @@ -29,6 +29,7 @@ appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += dcp_external_announce_test.o appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += dcp_external_relink_test.o appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += dcp_dropped_swap_test.o appledrm-y += iomfb_v26_6.o iomfb_v26_6_link.o j613_dcp_activate.o +appledrm-y += iomfb_v27.o appledrm-y += epic/dpavservep.o appledrm-y += plane.o diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index d7036e859293f..e8406f5111dfa 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -2,6 +2,8 @@ /* Copyright 2022 Sven Peter */ #include +#include +#include #include #include #include @@ -15,6 +17,8 @@ #include #include #include +#include +#include #include #include @@ -106,6 +110,7 @@ struct apple_dcp_afkep *afk_init(struct apple_dcp *dcp, u32 endpoint, afkep->ops = ops; afkep->dcp = dcp; + afkep->wire_v2 = dcp->fw_compat == DCP_FIRMWARE_V_27_0; afkep->endpoint = endpoint; afkep->wq = alloc_ordered_workqueue("apple-dcp-afkep%02x", WQ_MEM_RECLAIM, endpoint); @@ -1546,7 +1551,8 @@ static void afk_recv_handle_init(struct apple_dcp_afkep *ep, u32 channel, return; } ret = parse_epic_service_init(&ctx, &epic_name, &epic_class, &epic_unit, - ep->dcp->fw_compat == DCP_FIRMWARE_H17P); + (ep->dcp->fw_compat == DCP_FIRMWARE_H17P || ep->dcp->fw_compat == DCP_FIRMWARE_V_27_0), + ep->dcp->fw_compat == DCP_FIRMWARE_V_27_0); if (ret == -ENOENT && ep->dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G) { /* @@ -1649,6 +1655,8 @@ static void afk_recv_handle_init(struct apple_dcp_afkep *ep, u32 channel, } if (ops->prepare) afk_service_publish(service); + if (ops->init_props) + ops->init_props(service, epic_unit, props, props_size); if (ops->init) ops->init(service, epic_name, epic_class, epic_unit); dev_info(ep->dcp->dev, "AFK[ep:%02x]: new service %s on channel %d\n", @@ -1934,6 +1942,61 @@ static void afk_v2_status(struct apple_dcp_afkep *ep, dev_warn(ep->dcp->dev, "AFKV2 error reply failed: %d\n", ret); } +/* + * A buffer the DCP firmware names that is none of the host's pending command + * buffers. Only the firmware's own retained mappings are followed, with the + * permission they grant: a reply is written only where the firmware's leaf + * allows writes, and a DVA that Linux mapped itself, or that names Linux's RAM, + * a claimed resource or anything below DRAM (a device's registers, claimed or + * not), is refused. The copy goes through a write-combined mapping, as the + * RTKit buffers in the same firmware segments are mapped (dcp_rtk_shmem_setup()): + * the DCP is not described as DMA-coherent, so a cacheable alias could see + * stale data. + */ +static int afk_v27_access_remote(struct apple_dcp_afkep *ep, u64 dva, void *buffer, + size_t size, bool to_remote, bool copy) +{ + size_t off = 0; + + if (!dva || size > SZ_1M || size > U64_MAX - dva) + return -EINVAL; + while (off < size) { + phys_addr_t pa = 0; + size_t chunk = min_t(size_t, size - off, + PAGE_SIZE - ((dva + off) & ~PAGE_MASK)); + int ret = apple_dart_fw_remote(ep->dcp->dev, dva + off, to_remote, &pa); + void *cpu; + + if (!ret && (!dcp_fw_phys_in_dram(pa) || + region_intersects(pa, chunk, IORESOURCE_MEM, + IORES_DESC_NONE) != REGION_DISJOINT)) + ret = -EACCES; + if (ret) { + dev_err_ratelimited(ep->dcp->dev, + "AFK[ep:%02x]: refused remote buffer %#llx + %#zx (%s): %d\n", + ep->endpoint, dva, off, to_remote ? "reply" : "request", ret); + return ret; + } + if (!copy) { + off += chunk; + continue; + } + cpu = memremap(pa, chunk, MEMREMAP_WC); + if (!cpu) + return -ENOMEM; + if (to_remote) { + memcpy(cpu, buffer + off, chunk); + dma_wmb(); + } else { + dma_rmb(); + memcpy(buffer + off, cpu, chunk); + } + memunmap(cpu); + off += chunk; + } + return 0; +} + static int afk_v2_access_owned(struct apple_epic_service *service, u64 dva, void *buffer, size_t size, bool to_peer, bool copy) { @@ -1977,6 +2040,9 @@ static int afk_v2_access_owned(struct apple_epic_service *service, u64 dva, break; } spin_unlock_irqrestore(&service->lock, flags); + /* T6050 callbacks may refer to firmware-owned buffers in the retained DART. */ + if (ret && service->ep->dcp->fw_compat == DCP_FIRMWARE_V_27_0) + ret = afk_v27_access_remote(service->ep, dva, buffer, size, to_peer, copy); if (ret) dev_warn_ratelimited(service->ep->dcp->dev, "AFKV2 unowned buffer: ep=%u interface=%u size=%zu write=%u\n", @@ -2195,7 +2261,8 @@ static void afk_recv_handle(struct apple_dcp_afkep *ep, u32 channel, u32 type, hdr_len = sizeof(*ehdr) + sizeof(*c); if (data_size < hdr_len) goto too_small; - if (ep->dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G && + if (ep->dcp->fw_compat == DCP_FIRMWARE_H17P && + ep->dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G && !afk_validate_h17p_header(data, data_size)) { dev_err(ep->dcp->dev, "AFK[ep:%02x]: invalid compact message length\n", @@ -2213,7 +2280,7 @@ static void afk_recv_handle(struct apple_dcp_afkep *ep, u32 channel, u32 type, * queue channel at face value binds "system" to 0 and makes * every later announce and report look like a duplicate. */ - channel = le16_to_cpup((__le16 *)(data + 2)); + channel = get_unaligned_le16(data + 2); /* * Normalise into the wide sub-header the rest of this file @@ -2235,6 +2302,7 @@ static void afk_recv_handle(struct apple_dcp_afkep *ep, u32 channel, u32 type, payload = data + hdr_len; payload_size = data_size - hdr_len; + trace_afk_recv_handle(ep, channel, type, data_size, ehdr, eshdr); @@ -2579,7 +2647,7 @@ static int afk_send_epic_token(struct apple_dcp_afkep *ep, u32 channel, u16 tag, } spin_lock_irqsave(&ep->lock, flags); - if (ep->stopping) { + if (ep->stopping || !ep->txbfr.ready) { ret = -ESHUTDOWN; goto out; } @@ -2593,6 +2661,12 @@ static int afk_send_epic_token(struct apple_dcp_afkep *ep, u32 channel, u16 tag, dma_rmb(); rptr = le32_to_cpu(*ep->txbfr.rptr); wptr = le32_to_cpu(*ep->txbfr.wptr); + if (rptr >= ep->txbfr.bufsz || wptr >= ep->txbfr.bufsz || + !IS_ALIGNED(rptr, ep->txbfr.stride) || + !IS_ALIGNED(wptr, ep->txbfr.stride)) { + ret = -EPROTO; + goto out; + } trace_afk_send_rwptr_pre(ep, rptr, wptr); total_epic_size = sizeof(*ehdr) + (ep->wire_v2 ? sizeof(struct epic_sub_hdr_h17p) : sizeof(*eshdr)) + diff --git a/drivers/gpu/drm/apple/afk.h b/drivers/gpu/drm/apple/afk.h index 2586408eeebd8..44e4aac637a6d 100644 --- a/drivers/gpu/drm/apple/afk.h +++ b/drivers/gpu/drm/apple/afk.h @@ -119,6 +119,9 @@ struct apple_epic_service_ops { const char *class, s64 unit); void (*init)(struct apple_epic_service *service, const char *name, const char *class, s64 unit); + /* Consumed synchronously before init publishes service readiness. */ + void (*init_props)(struct apple_epic_service *service, s64 unit, + const void *props, size_t size); int (*call)(struct apple_epic_service *service, u32 idx, const void *data, size_t data_size, void *reply, size_t reply_size); diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index 02ef41d19947a..38d31dd0e110b 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -183,11 +183,12 @@ static int apple_drm_fbdev_probe(struct drm_fb_helper *helper, if (t8132_ext_crtc(crtc)) continue; dcp = platform_get_drvdata(to_apple_crtc(crtc)->dcp); - if (dcp->fw_compat == DCP_FIRMWARE_H17P) { + if (dcp->fw_compat == DCP_FIRMWARE_H17P || + dcp->fw_compat == DCP_FIRMWARE_V_27_0) { /* - * H17P applies the stored fourth byte as straight alpha, - * the H17G (M5) profile too: an XRGB console with X = 0 - * stays black. ARGB makes fbcon write it opaque. + * H17P and firmware 27 apply the stored fourth byte as + * straight alpha: an XRGB console with X = 0 stays black. + * ARGB makes fbcon write it opaque. */ sizes->surface_bpp = 32; sizes->surface_depth = 32; @@ -1684,8 +1685,13 @@ static int apple_drm_init(struct device *dev) * the boot framebuffer owns the screen; past the eviction there is no * way back. On refusal leave the DCP running (its shutdown sequence * can crash it and blank the panel), pin the module and keep the boot - * framebuffer. + * framebuffer. The pin is taken before the trials, while a refusal can + * still unwind; it is given back if every trial passes. */ + if (!try_module_get(THIS_MODULE)) { + ret = -ENODEV; + goto err_unbind; + } drm_for_each_crtc(crtc, &apple->drm) { const char *reason = "none"; @@ -1699,9 +1705,9 @@ static int apple_drm_init(struct device *dev) /* * The DCPs keep running and keep their pipelines: pin the * module and the DRM device their CRTCs and connectors live - * in, and keep them out of shutdown and sleep. + * in, and keep them out of shutdown and sleep. The module + * reference taken before the trials is kept. */ - __module_get(THIS_MODULE); drm_dev_get(&apple->drm); drm_for_each_crtc(crtc, &apple->drm) if (!t8132_ext_crtc(crtc)) @@ -1711,6 +1717,7 @@ static int apple_drm_init(struct device *dev) t8132_ext_drm_fini(&apple->drm); return 0; } + module_put(THIS_MODULE); if (retained_modeset) drm_modeset_lock_all(&apple->drm); diff --git a/drivers/gpu/drm/apple/dcp-fabric.c b/drivers/gpu/drm/apple/dcp-fabric.c index 3dc3128b600ae..eb11794f62cf3 100644 --- a/drivers/gpu/drm/apple/dcp-fabric.c +++ b/drivers/gpu/drm/apple/dcp-fabric.c @@ -2284,12 +2284,51 @@ int dcp_tunnel_crossbar_down(struct apple_dcp *dcp, u32 unit) * The output is the borrowed Type-C route's, or the fixed route's. Other * SoCs clock it at select. Runs inside a DCP apcall: only tb_lock. */ +static int dcp_direct_crossbar_link_t6050(struct apple_dcp *dcp, bool up) +{ + typeof(&apple_dp_phy_link_config) get_config; + typeof(&apple_dpxbar_link_configure) configure; + struct mux_control *xbar; + unsigned int pclk; + bool uhbr; + int ret; + + guard(mutex)(&dcp->tb_lock); + /* A borrowed Type-C route has its own mux; xbar remains the fixed + * output's mux (or NULL on a Type-C-only pipeline). */ + xbar = dcp->active_typec_route ? + dcp->active_typec_route->active_xbar : dcp->xbar; + if (dcp->dptx_tunnel || !dcp->phy || !xbar) + return 0; + get_config = symbol_get(apple_dp_phy_link_config); + if (!get_config) + return dcp->fw_compat == DCP_FIRMWARE_V_27_0 ? -ENODEV : 0; + ret = get_config(dcp->phy, &pclk, &uhbr); + symbol_put(apple_dp_phy_link_config); + if (ret == -EOPNOTSUPP) + return 0; + if (!up && ret == -ENOLINK) + ret = 0; + if (ret) + return ret; + if (!up) + return dcp_dpxbar_link(xbar, false); + configure = symbol_get(apple_dpxbar_link_configure); + if (!configure) + return -ENODEV; + ret = configure(xbar, pclk, uhbr); + symbol_put(apple_dpxbar_link_configure); + return ret ? ret : dcp_dpxbar_link(xbar, true); +} + int dcp_direct_crossbar_link(struct apple_dcp *dcp, bool up) { struct apple_dcp_typec_route *route; struct mux_control *xbar = NULL; int ret; + if (dcp->fw_compat == DCP_FIRMWARE_V_27_0) + return dcp_direct_crossbar_link_t6050(dcp, up); if (!of_machine_is_compatible("apple,t6030")) return 0; guard(mutex)(&dcp->tb_lock); @@ -4926,3 +4965,10 @@ void dcp_fabric_shutdown_dptx(struct apple_dcp *dcp) } } } + +int dcp_typec_test_link(struct platform_device *pdev, bool enabled) +{ + guard(mutex)(&dcp_typec_fabric_lock); + return enabled ? dcp_dptx_connect_oob(pdev, 0) : + dcp_dptx_disconnect_oob(pdev, 0); +} diff --git a/drivers/gpu/drm/apple/dcp-fabric.h b/drivers/gpu/drm/apple/dcp-fabric.h index f68b91e4d18f8..2db549172d1d5 100644 --- a/drivers/gpu/drm/apple/dcp-fabric.h +++ b/drivers/gpu/drm/apple/dcp-fabric.h @@ -127,4 +127,6 @@ int dcp_dptx_connect(struct apple_dcp *dcp, u32 port); int dcp_dptx_disconnect(struct apple_dcp *dcp, u32 port); bool dcp_is_external_h17p(struct apple_dcp *dcp); +int dcp_typec_test_link(struct platform_device *pdev, bool enabled); + #endif /* __APPLE_DCP_FABRIC_H__ */ diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 735e1f7d5054a..9573412f94b58 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -30,6 +31,7 @@ #include "dcp-180hz.h" #include "iomfb.h" #include "iomfb_h17p.h" +#include "iomfb_v27.h" #include "iomfb-state.h" #include "iomfb_v12_3.h" #include "iomfb_v13_3.h" @@ -37,6 +39,7 @@ #include "iomfb_v14_7_tpl.h" #include "iomfb_v26_6.h" #include "iomfb_v26_6_desktop.h" +#include "dcp-peer-frames-v27.h" #include "epic/dpavservep.h" #include "version_utils.h" @@ -233,6 +236,7 @@ enum dcp_iomfb_method_profile { }; struct apple_dcp_hw_data { + u8 firmware_dma_bits; u32 num_dptx_ports; enum dcp_iomfb_method_profile iomfb_method_profile; /* @@ -418,6 +422,10 @@ struct apple_dcp { struct apple_rtkit *rtk; struct apple_crtc *crtc; struct apple_connector *connector; + /* Protected by dcp_peer_lock; never take the physical route lock in + * an AFK callback (a route operation may be waiting for that reply). + */ + struct dcp_peer_frames_v27 peer_frames; struct apple_dcp_hw_data hw; bool external_prepared; @@ -453,7 +461,8 @@ struct apple_dcp { /* A desktop 26.x external processor (apple,t6041-dcpext) */ bool desktop_external; /* - * T8122 dcpext: the panel's DRM device never waits on this processor. + * A failsafe dcpext (dcp_ext_is_failsafe(): T8122's, T6041's, T6050's): + * the DRM device of the other displays never waits on this processor. * A probe or bind failure leaves it in the device as an unusable pipe * (ext_dead): no CRTC, Type-C routes disabled or ignored, RTKit * messages dropped. See dcp_platform_probe(). @@ -508,6 +517,8 @@ struct apple_dcp { /* DCP has crashed */ bool crashed; + /* CoreAnalytics events dumped in full (dcpep_cb_core_analytics_v27). */ + u32 analytics_dumps; DECLARE_BITMAP(iomfb_surfaces, DCP_MAX_PLANES); @@ -519,6 +530,18 @@ struct apple_dcp { /* clock rate request by dcp in */ struct clk *clk; struct clk *clk_194; + struct clk_bulk_data *firmware_clocks; + unsigned int num_firmware_clocks; + u64 disp_firmware_base[MAX_DISP_REGISTERS]; + u32 dfb_format, dfb_compression; + u64 dfb_region_size; + bool dfb_format_valid, dfb_compression_valid; + struct list_head iomfb_boolean_properties; + u32 iomfb_boolean_property_count; + u32 external_pipes_v27, external_displays_v27, external_suppression_v27; + struct dcp_swap_submit_req_v27 present_v27; + /* Both paired wire records belong to the primary endpoint lock. */ + struct dcp_swap_submit_req_v27 peer_present_v27; /* DCP shared memory */ void *shmem; @@ -632,6 +655,8 @@ struct apple_dcp { /* completion for active turning true */ struct completion start_done; + /* IOMFB has published fresh modes and acknowledged a Type-C display. */ + struct completion typec_display_ready; /* Is the DCP booted? */ bool active; @@ -851,6 +876,10 @@ struct apple_dcp { struct iomfb_transaction *active; struct task_struct *owner; struct work_struct work; + spinlock_t callback_lock; + struct list_head callbacks; + struct work_struct callback_work; + unsigned int callback_count; struct delayed_work timeout; struct delayed_work backlight_retry; unsigned long deadline; @@ -904,7 +933,8 @@ static inline bool iomfb_is_m4(const struct apple_dcp *dcp) /* Presents carry nits; one serialized IOMFB owner. */ static inline bool dcp_presents_nits(const struct apple_dcp *dcp) { - return (dcp->fw_compat == DCP_FIRMWARE_H17P && + return ((dcp->fw_compat == DCP_FIRMWARE_H17P || + dcp->fw_compat == DCP_FIRMWARE_V_27_0) && dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) || iomfb_is_m4(dcp); } @@ -913,6 +943,7 @@ static inline bool dcp_presents_nits(const struct apple_dcp *dcp) static inline bool dcp_fail_stop(const struct apple_dcp *dcp) { return dcp->fw_compat == DCP_FIRMWARE_H17P || + dcp->fw_compat == DCP_FIRMWARE_V_27_0 || iomfb_is_m4(dcp); } @@ -942,6 +973,8 @@ void iomfb_opaque_x_reset_h17p(struct apple_dcp *dcp); int iomfb_configure_backlight_h17p(struct apple_dcp *dcp, u32 maximum, bool inherited_valid, u32 inherited, bool default_valid, u32 default_nits); +int iomfb_queue_callback(struct apple_dcp *dcp, dcp_callback_t callback, + const void *data, size_t size, void *cookie); void iomfb_queue_init(struct apple_dcp *dcp); void iomfb_queue_stop(struct apple_dcp *dcp); bool iomfb_queue_drain(struct apple_dcp *dcp, unsigned long timeout); @@ -1068,6 +1101,33 @@ static inline bool dcp_ext_dptx_607(const struct apple_dcp *dcp) dcp->hw.fw_14_7_endpoints; } +/* + * Whether this DCP reads a surface's element geometry as the surface record + * defines it: bytes per element (pix_size, a linear plane's tile_size), the + * element's width and height in pixels, and compression information only for + * a compressed plane. Only firmware 27 (DCP_FIRMWARE_V_27_0, which only + * T6050's DCP hw data selects) is tested with it. Earlier firmware keeps the + * fixed 1/1/1 and has_compr_info 1 it has run on: before widening this, + * check a linear XRGB8888 and an NV12 swap, and the default framebuffer, on + * 12.x-26.x firmware. + */ +static inline bool dcp_surface_element_geometry(const struct apple_dcp *dcp) +{ + return dcp->fw_compat == DCP_FIRMWARE_V_27_0; +} + +/* + * Whether @pa, a physical address the DCP firmware placed a buffer at, can be + * DRAM. On Apple SoCs every device register window lies below DRAM, so an + * address below all of Linux's RAM may be a device's registers and is never + * mapped as memory. The firmware's own segments lie in DRAM beside Linux's RAM. + */ +static inline bool dcp_fw_phys_in_dram(phys_addr_t pa) +{ + return pa && region_intersects(0, pa, IORESOURCE_SYSTEM_RAM, + IORES_DESC_NONE) != REGION_DISJOINT; +} + #define DCP_AUDIO_MAX_CHANS 15 void dcp_trace_panel_property(struct apple_dcp *dcp, int tag, const char *key, diff --git a/drivers/gpu/drm/apple/dcp-peer-frames-v27.h b/drivers/gpu/drm/apple/dcp-peer-frames-v27.h new file mode 100644 index 0000000000000..abb7798a1764f --- /dev/null +++ b/drivers/gpu/drm/apple/dcp-peer-frames-v27.h @@ -0,0 +1,45 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef APPLE_DCP_PEER_FRAMES_V27_H +#define APPLE_DCP_PEER_FRAMES_V27_H + +#include + +/* Protected by dcp_peer_lock. Sequence comparisons cover less than 2^31 + * outstanding swaps, as in the existing single-pipe retirement path. + */ +struct dcp_peer_frames_v27 { + u32 first, submitted, engine[2], retired; + bool active, armed, failed, have[2], have_retired; +}; + +static inline void dcp_peer_frame_submit_v27(struct dcp_peer_frames_v27 *s, u32 id) +{ + if (!s->armed) { + s->first = id; + s->armed = true; + } + s->submitted = id; +} + +static inline bool dcp_peer_frame_complete_v27(struct dcp_peer_frames_v27 *s, + unsigned int engine, u32 id, + u32 *watermark) +{ + u32 done; + + if (!s->active || s->failed || !s->armed || engine > 1 || + (s32)(id - s->first) < 0 || (s32)(s->submitted - id) < 0 || + (s->have[engine] && (s32)(id - s->engine[engine]) <= 0)) + return false; + s->engine[engine] = id; + s->have[engine] = true; + if (!s->have[0] || !s->have[1]) + return false; + done = (s32)(s->engine[0] - s->engine[1]) < 0 ? s->engine[0] : s->engine[1]; + if (s->have_retired && (s32)(done - s->retired) <= 0) + return false; + s->retired = *watermark = done; + s->have_retired = true; + return true; +} +#endif diff --git a/drivers/gpu/drm/apple/dcp-peer-modeset-probe-v27.inc b/drivers/gpu/drm/apple/dcp-peer-modeset-probe-v27.inc new file mode 100644 index 0000000000000..a5aa449b446dd --- /dev/null +++ b/drivers/gpu/drm/apple/dcp-peer-modeset-probe-v27.inc @@ -0,0 +1,227 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* Paired modeset bring-up, opt-in only. Scanout is a separate step. */ +/* The power-state step starts one asynchronous AP request per engine. + * Submit both before waiting: either firmware may rendezvous with its peer. + */ +static int dcp_peer_probe_power(struct apple_dcp *dcp, struct apple_dcp *peer, + bool on) +{ + struct dcp_peer_video_wait *wait; + struct dcp_method_entry method = { .name = "peer_power_test", + .tag = {'A', '4', '8', '4'} }; + u8 power[16] = {}, response[8]; + int ret; + + wait = kzalloc(sizeof(*wait), GFP_KERNEL); + if (!wait) + return -ENOMEM; + kref_init(&wait->refs); + init_completion(&wait->done); + wait->size = sizeof(response); + put_unaligned_le64(on, power); + kref_get(&wait->refs); + dcp_push(dcp, false, &method, sizeof(power), sizeof(response), power, + dcp_peer_video_done, wait); + ret = dcp_peer_video_call(peer, "A484", power, sizeof(power), + response, sizeof(response)); + if (!ret && get_unaligned_le32(response + 4)) + ret = -EIO; + if (!wait_for_completion_timeout(&wait->done, msecs_to_jiffies(2000))) + ret = ret ?: -ETIMEDOUT; + else if (!ret && get_unaligned_le32(wait->reply + 4)) + ret = -EIO; + kref_put(&wait->refs, dcp_peer_video_free); + dev_info(dcp->dev, "peer power %u result=%d\n", on, ret); + return ret; +} + +static int dcp_peer_probe_quiesce(struct apple_dcp *dcp, struct apple_dcp *peer) +{ + __le32 manage = 0; + int ret; + + /* Setting the digital-out mode: pair API power0, 100ms, + * then manage_elements(false), A362, on each engine before A365. + * Only discard AP state after both engines have acknowledged power0. + */ + { + guard(mutex)(&dcp_peer_lock); + dcp->peer_tiling.modeset_complete = peer->peer_tiling.modeset_complete = false; + } + ret = dcp_peer_probe_power(dcp, peer, false); + if (ret) + return ret; + msleep(100); + ret = dcp_peer_video_call(dcp, "A362", &manage, sizeof(manage), NULL, 0); + if (ret) + return ret; + ret = dcp_peer_video_call(peer, "A362", &manage, sizeof(manage), NULL, 0); + if (ret) + return ret; + { + guard(mutex)(&dcp_peer_lock); + dcp->peer_frames = (struct dcp_peer_frames_v27){}; + dcp->peer_tiling.member = peer->peer_tiling.member = false; + dcp->peer_tiling.vftg = peer->peer_tiling.vftg = false; + dcp->peer_tiling.modeset_complete = peer->peer_tiling.modeset_complete = false; + dcp->peer_tiling.phase = peer->peer_tiling.phase = 0; + dcp->peer_tiling.modeset_state = peer->peer_tiling.modeset_state = 0; + dcp->peer_tiling.ready = peer->peer_tiling.ready = false; + } + return 0; +} + +static int dcp_peer_probe_prepare_vftg(struct apple_dcp *dcp, struct apple_dcp *peer, + const struct dcp_peer_insert_video_req_v27 *video) +{ + guard(mutex)(&dcp_peer_lock); + if (dcp->peer_reserved != peer || peer->peer_owner != dcp) + return -ENODEV; + /* These are set BEFORE A366/A365, but membership is marked only after + * both merge-mode insertions have accepted the derived video data. + */ + dcp->peer_tiling.vftg = peer->peer_tiling.vftg = true; + dcp->peer_tiling.modeset_complete = peer->peer_tiling.modeset_complete = false; + dcp->peer_tiling.primary = true; + peer->peer_tiling.primary = false; + dcp->peer_tiling.width = peer->peer_tiling.width = + get_unaligned_le32(video->video.timing + 8); + dcp->peer_tiling.height = peer->peer_tiling.height = + get_unaligned_le32(video->video.timing + 0x28); + dcp->peer_tiling.x = dcp->peer_tiling.y = peer->peer_tiling.y = 0; + peer->peer_tiling.x = dcp->peer_tiling.width; + return 0; +} + +static int dcp_peer_probe_copy_dsc(struct apple_dcp *primary, struct apple_dcp *peer) +{ + u8 caps_req[36] = {}, caps[32], bpp_req[4] = {}, bpp[8]; + u8 hints_req[12] = {}, hints[12]; + u32 required; + int ret; + + ret = dcp_peer_video_call_stream(primary, true, "A369", NULL, 0, &required, 4); + if (ret || !(required & 1)) + return ret; + ret = dcp_peer_video_call_stream(primary, true, "A367", caps_req, 36, caps, 32); + if (ret) + return ret; + memcpy(caps_req, caps, 32); + ret = dcp_peer_video_call(peer, "A368", caps_req, 36, caps, 32); + if (ret) + return ret; + ret = dcp_peer_video_call_stream(primary, true, "A370", bpp_req, 4, bpp, 8); + if (ret || get_unaligned_le32(bpp + 2)) + return ret ?: -EIO; + memcpy(bpp_req, bpp, 2); + ret = dcp_peer_video_call(peer, "A371", bpp_req, 4, bpp, 8); + if (ret || get_unaligned_le32(bpp + 2)) + return ret ?: -EIO; + ret = dcp_peer_video_call_stream(primary, true, "A372", hints_req, 12, hints, 12); + if (ret || get_unaligned_le32(hints + 8)) + return ret ?: -EIO; + memcpy(hints_req, hints, 8); + ret = dcp_peer_video_call(peer, "A373", hints_req, 12, hints, 12); + if (ret || get_unaligned_le32(hints + 8)) + return ret ?: -EIO; + dev_info(primary->dev, "peer DSC copied: bpp=%u hints=%*ph\n", + get_unaligned_le16(bpp), 8, hints); + return 0; +} + +static int dcp_peer_probe_modeset(struct apple_dcp *dcp, struct apple_dcp *peer, + const struct dcp_peer_insert_video_req_v27 *video, + bool firmware_requires_merge) +{ + struct dcp_peer_video_wait *wait; + struct dcp_method_entry method = { .name = "peer_modeset_test", + .tag = {'A', '4', '1', '3'} }; + __le32 mode[2] = {video->color_id, video->timing_id}; + u32 primary_id, peer_id, phase, status; + unsigned long deadline; + int ret; + + { + guard(mutex)(&dcp_peer_lock); + if (dcp->peer_reserved != peer || peer->peer_owner != dcp || + !dcp->peer_tiling.display_id_valid || !peer->peer_tiling.display_id_valid) + return -ENODEV; + dcp->peer_tiling.member = peer->peer_tiling.member = true; + /* The original A364 result is preserved even when the host + * allocation requires a pair for a lower-bandwidth timing. + */ + dcp->peer_tiling.mode_flag = peer->peer_tiling.mode_flag = + !firmware_requires_merge; + } + ret = dcp_peer_probe_power(dcp, peer, true); + if (ret) + return ret; + /* Wait 100ms after both API-power transitions, before swap IDs. */ + msleep(100); + ret = dcp_peer_video_call(dcp, "A360", NULL, 0, &primary_id, 4); + if (ret) + return ret; + ret = dcp_peer_video_call(peer, "A360", NULL, 0, &peer_id, 4); + if (ret) + return ret; + if (primary_id != peer_id) { + status = max(primary_id, peer_id); + ret = dcp_peer_video_call(primary_id < peer_id ? dcp : peer, + "A361", &status, 4, NULL, 0); + if (ret) + return ret; + } + phase = 1; + dcp_peer_tiling_state_v27(dcp, true, 1, 10, &phase); + dcp_peer_tiling_state_v27(peer, true, 1, 10, &phase); + wait = kzalloc(sizeof(*wait), GFP_KERNEL); + if (!wait) + return -ENOMEM; + init_completion(&wait->done); + kref_init(&wait->refs); + kref_get(&wait->refs); + wait->size = 4; + dcp_push(dcp, false, &method, sizeof(mode), 4, mode, + dcp_peer_video_done, wait); + /* The primary negotiates the link, signals phase2, and then waits for + * its peer. Waiting for its final ACK first would deadlock the pair. + */ + deadline = jiffies + msecs_to_jiffies(8500); + for (;;) { + phase = 0; + if (!dcp_peer_tiling_state_v27(dcp, false, 1, 10, &phase)) { + ret = -ENODEV; + goto put; + } + if (phase >= 2) + break; + if (completion_done(&wait->done) || time_after_eq(jiffies, deadline)) { + ret = completion_done(&wait->done) ? -EIO : -ETIMEDOUT; + dev_err(dcp->dev, "peer primary ended before phase2: phase=%u result=%#x\n", + phase, get_unaligned_le32(wait->reply)); + goto put; + } + msleep(2); + } + dev_info(dcp->dev, "peer primary reached modeset phase%u\n", phase); + ret = dcp_peer_probe_copy_dsc(dcp, peer); + if (ret) + goto put; + ret = dcp_peer_video_call(peer, "A413", mode, sizeof(mode), &status, 4); + if (ret || status) { + ret = ret ?: -EIO; + goto put; + } + if (!wait_for_completion_timeout(&wait->done, msecs_to_jiffies(8500))) + ret = -ETIMEDOUT; + else if (get_unaligned_le32(wait->reply)) + ret = -EIO; +put: + kref_put(&wait->refs, dcp_peer_video_free); + if (!ret) { + guard(mutex)(&dcp_peer_lock); + dcp->peer_tiling.modeset_complete = peer->peer_tiling.modeset_complete = true; + } + dev_info(dcp->dev, "peer paired modeset result=%d (scanout not submitted)\n", ret); + return ret; +} diff --git a/drivers/gpu/drm/apple/dcp-peer-probe-v27.inc b/drivers/gpu/drm/apple/dcp-peer-probe-v27.inc new file mode 100644 index 0000000000000..33def11c752ea --- /dev/null +++ b/drivers/gpu/drm/apple/dcp-peer-probe-v27.inc @@ -0,0 +1,98 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* Explicit, opt-in firmware bring-up command. This is not the final hotplug + * lifecycle: use only the dedicated, otherwise unrouted peer reservation. + */ +struct dcp_peer_probe_wait { + struct kref refs; + struct completion done; +}; + +static void dcp_peer_probe_free(struct kref *ref) +{ + kfree(container_of(ref, struct dcp_peer_probe_wait, refs)); +} + +static void dcp_peer_probe_done(struct apple_dcp *dcp, void *out, void *cookie) +{ + struct dcp_peer_probe_wait *wait = cookie; + + complete(&wait->done); + kref_put(&wait->refs, dcp_peer_probe_free); +} + +static ssize_t peer_setup_test_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + static const struct dcp_method_entry method = { + .name = "setup_peer_dcp", .tag = {'A', '3', '6', '6'}, + }; + struct apple_dcp *dcp = dev_get_drvdata(dev); + struct dcp_peer_probe_wait *wait; + __le32 enabled = cpu_to_le32(1); + bool value; + long ret; + + if (kstrtobool(buf, &value) || !value) + return -EINVAL; + if (!peer_allocation_probe || !dcp) + return -EOPNOTSUPP; + wait = kzalloc(sizeof(*wait), GFP_KERNEL); + if (!wait) + return -ENOMEM; + init_completion(&wait->done); + kref_init(&wait->refs); + { + guard(mutex)(&dcp_peer_lock); + if (!dcp->peer_owner || dcp->peer_owner->peer_reserved != dcp || + dcp->peer_unbinding || dcp->crashed || !dcp->active || + dcp->fw_compat != DCP_FIRMWARE_V_27_0 || + dcp->fixed_phy || dcp->nr_typec_routes || + (dcp->connector && dcp->connector->connected) || + dcp->ch_cmd.depth || dcp->ch_cb.depth || dcp->ch_async.depth || + dcp->ch_oobcmd.depth || dcp->ch_oobcb.depth || + dcp->ch_oobasync.depth) { + kref_put(&wait->refs, dcp_peer_probe_free); + return -EBUSY; + } + kref_get(&wait->refs); + /* Peer-DCP setup (one bool argument): four input + * bytes (boolean followed by zero padding), no output. + */ + dcp_push(dcp, false, &method, sizeof(enabled), 0, &enabled, + dcp_peer_probe_done, wait); + } + ret = wait_for_completion_timeout(&wait->done, msecs_to_jiffies(2000)); + kref_put(&wait->refs, dcp_peer_probe_free); + dev_info(dev, "peer A366 setup test %s\n", ret ? "acknowledged" : "timed out"); + return ret ? count : -ETIMEDOUT; +} +static DEVICE_ATTR_WO(peer_setup_test); + +#include "dcp-peer-video-probe-v27.inc" +#include "dcp-peer-swap-probe-v27.inc" +#include "dcp-peer-runtime-v27.inc" + +static struct attribute *dcp_peer_probe_attrs[] = { + &dev_attr_peer_setup_test.attr, + &dev_attr_peer_video_test.attr, + &dev_attr_peer_link_test.attr, + &dev_attr_peer_swap_test.attr, + &dev_attr_peer_stop_test.attr, + NULL, +}; + +static umode_t dcp_peer_probe_visible(struct kobject *kobj, + struct attribute *attr, int n) +{ + return peer_allocation_probe ? attr->mode : 0; +} + +static const struct attribute_group dcp_peer_probe_group = { + .attrs = dcp_peer_probe_attrs, + .is_visible = dcp_peer_probe_visible, +}; +static const struct attribute_group *dcp_peer_probe_groups[] = { + &dcp_peer_probe_group, + NULL, +}; diff --git a/drivers/gpu/drm/apple/dcp-peer-runtime-v27.inc b/drivers/gpu/drm/apple/dcp-peer-runtime-v27.inc new file mode 100644 index 0000000000000..b1f99d6ddd651 --- /dev/null +++ b/drivers/gpu/drm/apple/dcp-peer-runtime-v27.inc @@ -0,0 +1,161 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* Kept opt-in while paired DRM lifecycle is being validated on T6050. */ +static bool peer_scanout; +module_param(peer_scanout, bool, 0444); +MODULE_PARM_DESC(peer_scanout, "Enable firmware27 paired DRM scanout bring-up"); + +static bool dcp_peer_transport_idle(struct apple_dcp *dcp) +{ + return dcp->active && !dcp->crashed && !dcp->peer_unbinding && + !dcp->ch_cmd.depth && !dcp->ch_cb.depth && !dcp->ch_async.depth && + !dcp->ch_oobcmd.depth && !dcp->ch_oobcb.depth && !dcp->ch_oobasync.depth; +} + +int dcp_peer_modeset_v27(struct apple_dcp *dcp, u32 timing, u32 color, bool vrr) +{ + struct apple_dcp *peer; + int ret = -EOPNOTSUPP; + + if (!peer_scanout || !peer_allocation_probe || + dcp->fw_compat != DCP_FIRMWARE_V_27_0) + return ret; + if (timing > U32_MAX - 10000 || color > U32_MAX - 10000) + return -EINVAL; + if (!device_trylock(dcp->dev)) + return -EBUSY; + { + guard(mutex)(&dcp_peer_lock); + peer = dcp->peer_reserved; + if (!peer) + goto unlock; + if (peer->peer_owner != dcp || !dcp_peer_transport_idle(dcp) || + !dcp_peer_transport_idle(peer) || peer->fixed_phy || + peer->nr_typec_routes || !peer->crtc || + !peer->crtc->base.state || peer->crtc->base.state->active || + !device_trylock(peer->dev)) { + ret = -EBUSY; + goto unlock; + } + } + /* Do not silently select an untiled mode when paired VRR was requested. + * The internal and single-pipe external adaptive-sync paths are unchanged. + */ + if (vrr) { + struct dcp_peer_get_video_req_v27 req = { + .timing_id = cpu_to_le32(timing), .color_id = cpu_to_le32(color), + }; + struct dcp_peer_get_video_resp_v27 resp; + + ret = dcp_peer_video_call(dcp, "A364", &req, sizeof(req), &resp, sizeof(resp)); + if (ret || !(resp.success & 1) || (resp.requires_merge & 1)) { + ret = ret ?: -EINVAL; + goto peer_unlock; + } + } + ret = dcp_peer_install_video_v27(dcp, peer, timing, color, 4); + if (!ret) { + guard(mutex)(&dcp_peer_lock); + dcp->peer_frames = (struct dcp_peer_frames_v27){ .active = true }; + dev_info(dcp->dev, "paired DRM mode ready: timing=%u color=%u\n", timing, color); + } +peer_unlock: + device_unlock(peer->dev); +unlock: + device_unlock(dcp->dev); + return ret; +} + +/* Return1 after a paired stop,0 for a normal CRTC, or an error. */ +int dcp_peer_stop_v27(struct apple_dcp *dcp) +{ + struct apple_dcp *peer; + int ret = 0; + + if (!peer_scanout || !READ_ONCE(dcp->peer_frames.active)) + return 0; + if (!device_trylock(dcp->dev)) + return -EBUSY; + { + guard(mutex)(&dcp_peer_lock); + peer = dcp->peer_reserved; + if (!peer || peer->peer_owner != dcp || !dcp_peer_transport_idle(dcp) || + !dcp_peer_transport_idle(peer) || !device_trylock(peer->dev)) { + ret = -EBUSY; + goto unlock; + } + } + ret = dcp_peer_probe_quiesce(dcp, peer); + device_unlock(peer->dev); + if (!ret) + ret = 1; +unlock: + device_unlock(dcp->dev); + return ret; +} + +struct apple_dcp *dcp_peer_scanout_v27(struct apple_dcp *dcp) +{ + struct apple_dcp *peer; + + if (!peer_scanout || !READ_ONCE(dcp->peer_frames.active)) + return NULL; + guard(mutex)(&dcp_peer_lock); + if (!dcp->peer_frames.active) + return NULL; + peer = dcp->peer_reserved; + if (!peer || peer->peer_owner != dcp || dcp->peer_frames.failed || + !dcp->peer_tiling.modeset_complete || !peer->peer_tiling.modeset_complete || + dcp->crashed || peer->crashed || dcp->peer_unbinding || peer->peer_unbinding) + return ERR_PTR(-EIO); + return peer; +} + +void dcp_peer_swap_armed_v27(struct apple_dcp *dcp, u32 id) +{ + guard(mutex)(&dcp_peer_lock); + dcp_peer_frame_submit_v27(&dcp->peer_frames, id); +} + +void dcp_peer_swap_failed_v27(struct apple_dcp *dcp) +{ + guard(mutex)(&dcp_peer_lock); + dcp->peer_frames.failed = true; + dcp_mode_invalidate(&dcp->mode_state); +} + +bool dcp_peer_swap_complete_v27(struct apple_dcp *dcp, u32 id, ktime_t now, + void (*report)(struct apple_dcp *, u32, ktime_t)) +{ + struct apple_dcp *owner; + u32 watermark; + + /* Independent CRTCs must not serialize through the pair's mutex. */ + if (!peer_scanout || (!READ_ONCE(dcp->peer_frames.active) && + !READ_ONCE(dcp->peer_owner))) + return false; + guard(mutex)(&dcp_peer_lock); + owner = dcp->peer_owner ?: dcp; + if (!peer_scanout || !owner->peer_frames.active) + return false; + if (dcp_peer_frame_complete_v27(&owner->peer_frames, dcp != owner, id, &watermark)) + /* Queue the combined watermark to the primary IOMFB owner. */ + report(owner, watermark, now); + return true; +} + +int dcp_peer_color_v27(struct apple_dcp *dcp, struct iomfb_set_matrix_req *matrix) +{ + struct apple_dcp *peer = dcp_peer_scanout_v27(dcp); + u32 status, identity = 0x10000; + int ret; + + if (!peer) + return 0; + if (IS_ERR(peer)) + return PTR_ERR(peer); + ret = dcp_peer_video_call(peer, "A423", matrix, sizeof(*matrix), &status, 4); + if (ret || status) + return ret ?: -EIO; + ret = dcp_peer_video_call(peer, "A428", &identity, sizeof(identity), &status, 4); + return ret ?: (status ? -EIO : 0); +} diff --git a/drivers/gpu/drm/apple/dcp-peer-swap-probe-v27.inc b/drivers/gpu/drm/apple/dcp-peer-swap-probe-v27.inc new file mode 100644 index 0000000000000..f6c8ced4317ed --- /dev/null +++ b/drivers/gpu/drm/apple/dcp-peer-swap-probe-v27.inc @@ -0,0 +1,243 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* Explicit background-only paired swap diagnostic. No framebuffer DMA or + * DRM events are owned here. Both DRM CRTCs must remain inactive. + */ +struct dcp_peer_background { + struct iomfb_transaction transaction; + struct dcp_peer_video_wait *wait; + struct dcp_swap_submit_req_v27 wire; + bool sent; +}; + +static void dcp_peer_background_done(struct apple_dcp *dcp, void *out, void *cookie) +{ + struct dcp_peer_background *background = cookie; + struct dcp_swap_submit_resp_v27 failed = { .ret = ~0U }; + struct dcp_swap_submit_resp_v27 *response = out ?: &failed; + + if (!dcp_present_submit_h17p(&dcp->present_state_h17p, + background->wire.swap.swap_id, !response->ret)) + WRITE_ONCE(dcp->crashed, true); + dcp_peer_video_done(dcp, response, background->wait); +} + +static void dcp_peer_background_start(struct apple_dcp *dcp, + struct iomfb_transaction *transaction) +{ + struct dcp_peer_background *background = + container_of(transaction, struct dcp_peer_background, transaction); + struct dcp_method_entry method = { .name = "peer_background_test", + .tag = {'A', '4', '0', '8'} }; + struct dcp_swap_submit_resp_v27 failed = { .ret = ~0U }; + + background->sent = true; + if (!dcp_present_begin_h17p(&dcp->present_state_h17p, + background->wire.swap.swap_id)) { + dcp_peer_video_done(dcp, &failed, background->wait); + return; + } + dcp_push(dcp, false, &method, sizeof(background->wire), sizeof(failed), + &background->wire, dcp_peer_background_done, background); +} + +static void dcp_peer_background_release(struct iomfb_transaction *transaction) +{ + struct dcp_peer_background *background = + container_of(transaction, struct dcp_peer_background, transaction); + + if (!background->sent) + kref_put(&background->wait->refs, dcp_peer_video_free); + kfree(background); +} + +static int dcp_peer_queue_background(struct apple_dcp *dcp, + struct dcp_swap_submit_req_v27 *wire, + struct dcp_peer_video_wait *wait) +{ + struct dcp_peer_background *background; + int ret; + + background = kzalloc_obj(*background); + if (!background) + return -ENOMEM; + background->wire = *wire; + background->wait = wait; + kref_get(&wait->refs); + background->transaction.start = dcp_peer_background_start; + background->transaction.release = dcp_peer_background_release; + ret = iomfb_queue(dcp, &background->transaction); + if (ret) + dcp_peer_background_release(&background->transaction); + return ret; +} + +static int dcp_peer_probe_background(struct apple_dcp *dcp, struct apple_dcp *peer) +{ + struct dcp_swap_start_req_h17p start = {}; + struct dcp_swap_start_resp_h17p a, b; + struct dcp_swap_submit_req_v27 *submit; + struct dcp_swap_submit_resp_v27 response; + struct dcp_peer_video_wait *wait, *peer_wait; + unsigned long deadline; + int ret; + + submit = kzalloc(sizeof(*submit), GFP_KERNEL); + wait = kzalloc(sizeof(*wait), GFP_KERNEL); + peer_wait = kzalloc(sizeof(*peer_wait), GFP_KERNEL); + if (!submit || !wait || !peer_wait) { + kfree(submit); + kfree(wait); + kfree(peer_wait); + return -ENOMEM; + } + kref_init(&wait->refs); + init_completion(&wait->done); + wait->size = sizeof(response); + kref_init(&peer_wait->refs); + init_completion(&peer_wait->done); + peer_wait->size = sizeof(response); + ret = dcp_peer_video_call(dcp, "A406", &start, sizeof(start), &a, sizeof(a)); + if (ret || a.ret) { + ret = ret ?: -EIO; + goto out; + } + ret = dcp_peer_video_call(peer, "A406", &start, sizeof(start), &b, sizeof(b)); + if (ret || b.ret || a.swap_id != b.swap_id) { + dev_err(dcp->dev, "peer swap-start mismatch/error: %u/%u result=%d\n", + a.swap_id, ret ? 0 : b.swap_id, ret); + ret = ret ?: -EIO; + goto out; + } + submit->swap.swap_id = a.swap_id; + submit->swap.swap_enabled = submit->swap.swap_completed = IOMFB_SET_BACKGROUND | 7; + submit->swap.bg_color = 0xffff0000; + memset(submit->surf_null, 1, sizeof(submit->surf_null)); + submit->unknown_pointer_null = submit->unknown_output_null = true; + ret = dcp_peer_queue_background(dcp, submit, wait); + if (ret) + goto out; + /* Queue both engines before waiting, with each present tracked by its + * own IOMFB transaction and completion state. */ + submit->swap.bg_color = 0xff0000ff; + ret = dcp_peer_queue_background(peer, submit, peer_wait); + if (ret) + goto out; + if (!wait_for_completion_timeout(&peer_wait->done, msecs_to_jiffies(2000))) { + ret = -ETIMEDOUT; + goto out; + } + memcpy(&response, peer_wait->reply, sizeof(response)); + if (response.ret) { + ret = -EIO; + goto out; + } + if (!wait_for_completion_timeout(&wait->done, msecs_to_jiffies(2000))) { + ret = -ETIMEDOUT; + goto out; + } + memcpy(&response, wait->reply, sizeof(response)); + if (response.ret) { + ret = -EIO; + goto out; + } + deadline = jiffies + msecs_to_jiffies(2000); + while (!READ_ONCE(dcp->have_swap_complete) || !READ_ONCE(peer->have_swap_complete) || + READ_ONCE(dcp->last_swap_id) != a.swap_id || + READ_ONCE(peer->last_swap_id) != a.swap_id) { + if (time_after_eq(jiffies, deadline)) { + ret = -ETIMEDOUT; + goto out; + } + msleep(2); + } + dev_info(dcp->dev, "peer background swap %u completed on both engines (optical unverified)\n", + a.swap_id); +out: + dev_info(dcp->dev, "peer background test result=%d\n", ret); + kref_put(&wait->refs, dcp_peer_video_free); + kref_put(&peer_wait->refs, dcp_peer_video_free); + kfree(submit); + return ret; +} + +static ssize_t peer_swap_test_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct apple_dcp *dcp, *peer; + struct device *peer_dev = NULL; + bool enabled; + int ret; + + if (kstrtobool(buf, &enabled) || !enabled) + return -EINVAL; + if (!peer_allocation_probe || !device_trylock(dev)) + return -EBUSY; + dcp = dev_get_drvdata(dev); + { + guard(mutex)(&dcp_peer_lock); + peer = dcp ? dcp->peer_reserved : NULL; + if (!peer || peer->peer_owner != dcp || !dcp_peer_video_idle(dcp) || + !dcp_peer_video_idle(peer) || READ_ONCE(dcp->peer_frames.active) || !dcp->peer_tiling.member || + !peer->peer_tiling.member || !dcp->peer_tiling.modeset_complete || + !peer->peer_tiling.modeset_complete || !dcp->crtc || !peer->crtc || + !dcp->crtc->base.state || !peer->crtc->base.state || + dcp->crtc->base.state->active || peer->crtc->base.state->active || + dcp->fw_compat != DCP_FIRMWARE_V_27_0 || + peer->fw_compat != DCP_FIRMWARE_V_27_0 || + !device_trylock(peer->dev)) { + ret = -EBUSY; + goto out; + } + peer_dev = peer->dev; + } + ret = dcp_peer_probe_background(dcp, peer); +out: + if (peer_dev) + device_unlock(peer_dev); + device_unlock(dev); + return ret ? ret : count; +} +static DEVICE_ATTR_WO(peer_swap_test); + +/* End a diagnostic pair using the firmware's power/element sequence. Keep the + * reservation on failure: a failed ACK is not evidence that DMA has stopped. + */ +static ssize_t peer_stop_test_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct apple_dcp *dcp, *peer; + struct device *peer_dev = NULL; + bool enabled; + int ret; + + if (kstrtobool(buf, &enabled) || !enabled) + return -EINVAL; + if (!peer_allocation_probe || !device_trylock(dev)) + return -EBUSY; + dcp = dev_get_drvdata(dev); + { + guard(mutex)(&dcp_peer_lock); + peer = dcp ? dcp->peer_reserved : NULL; + if (!peer || peer->peer_owner != dcp || !dcp_peer_video_idle(dcp) || + !dcp_peer_video_idle(peer) || !dcp->crtc || !peer->crtc || + !dcp->crtc->base.state || !peer->crtc->base.state || + dcp->crtc->base.state->active || peer->crtc->base.state->active || + dcp->fw_compat != DCP_FIRMWARE_V_27_0 || + peer->fw_compat != DCP_FIRMWARE_V_27_0 || + !device_trylock(peer->dev)) { + ret = -EBUSY; + goto out; + } + peer_dev = peer->dev; + } + ret = dcp_peer_probe_quiesce(dcp, peer); + dev_info(dev, "peer stop test result=%d\n", ret); +out: + if (peer_dev) + device_unlock(peer_dev); + device_unlock(dev); + return ret ? ret : count; +} +static DEVICE_ATTR_WO(peer_stop_test); diff --git a/drivers/gpu/drm/apple/dcp-peer-v27.h b/drivers/gpu/drm/apple/dcp-peer-v27.h new file mode 100644 index 0000000000000..cea2a4f13e43b --- /dev/null +++ b/drivers/gpu/drm/apple/dcp-peer-v27.h @@ -0,0 +1,125 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef APPLE_DCP_PEER_V27_H +#define APPLE_DCP_PEER_V27_H + +#include +#include +#include + +#include "dcp-peer-v26_6.h" + +struct dcp_dptx_caps { + u32 width, total_rate, active_rate; +}; + +/* Per-engine state used by D114/D115 in a VFTG pair. This is not + * display-wall emulation: membership is set only while a split mode is active. + */ +/* The record is struct dcp_peer_tiling, shared with the desktop 26.6 pair. */ + +static inline bool dcp_peer_tiling_get_v27( + const struct dcp_peer_tiling *own, + const struct dcp_peer_tiling *related, u32 event, u32 param, u32 *value) +{ + const struct dcp_peer_tiling *target = own; + + /* The getter redirects event 2 only for an active pair. */ + if (event == 2 && own->member) { + if (!related) + return false; + target = related; + } + switch (param) { + case 1: *value = target->member; return true; + case 2: + if (event == 1 || target->member) + *value = target->phase; + return true; + case 3: return true; /* The non-wall case preserves the input. */ + case 4: + if (event != 1 && !target->member) + return true; + if (!target->display_id_valid) + return false; + *value = target->display_id; + return true; + case 5: case 12: *value = target->x; return true; + case 6: + if (event == 1 || target->member) + *value = target->ready; + return true; + case 8: *value = own->vftg; return true; + case 9: *value = own->primary; return true; + case 10: + if (event == 1 || target->member) + *value = target->modeset_state; + return true; + case 11: *value = 0; return true; /* This pair is not a display wall. */ + case 13: *value = target->y; return true; + case 14: *value = target->mode_flag; return true; + default: return false; + } +} + +static inline bool dcp_peer_tiling_set_v27(struct dcp_peer_tiling *own, + struct dcp_peer_tiling *related, u32 event, u32 param, u32 value) +{ + /* Unlike the getter, the setter always redirects event 2. */ + struct dcp_peer_tiling *target = event == 2 ? related : own; + + if (!target) + return false; + switch (param) { + case 2: target->phase = value; return true; + case 6: target->ready = !!value; return true; + case 10: target->modeset_state = value; return true; + default: return false; + } +} + +/* AV display hints and display allocation records. */ +struct dcp_display_hints_v27 { + u8 hints[0x6c]; + u8 padding[4]; + __le32 primary_id, peer_id; + u8 extra_pipes, flags; + u8 allocation_reserved[22]; + __le32 status; + u8 tail[12]; +} __packed; + +static_assert(sizeof(struct dcp_display_hints_v27) == 160); +static_assert(offsetof(struct dcp_display_hints_v27, primary_id) == 0x70); +static_assert(offsetof(struct dcp_display_hints_v27, status) == 0x90); + +/* Extra pipe count, as the T602X display crossbar computes it. + * Caps come from the live DPTX service, never from a forced clock limit. + */ +static inline u32 dcp_peer_extra_pipes(const u8 *hints, + const struct dcp_dptx_caps *caps) +{ + u32 width = get_unaligned_le32(hints + 0x38); + u32 total = get_unaligned_le32(hints + 0x40); + u32 active = get_unaligned_le32(hints + 0x44); + u32 extra; + + if (!(hints[0] & 1) || !width || !total || !active || + !caps->width || !caps->total_rate || !caps->active_rate) + return 0; + extra = (width - 1) / caps->width; + extra = max(extra, (total - 1) / caps->total_rate); + return max(extra, (active - 1) / caps->active_rate); +} + +/* Peer validity rule of the T8132 display crossbar, inherited by T6050. + * Endpoint zero is the internal display; a peer must be on the same die. + */ +static inline bool dcp_peer_index_allowed(u32 primary, u32 peer) +{ + static const u8 masks[] = { 0, 0x1c, 0x18, 0x10, 0 }; + + return primary < ARRAY_SIZE(masks) && peer < ARRAY_SIZE(masks) && + (masks[primary] & BIT(peer)); +} + +#endif diff --git a/drivers/gpu/drm/apple/dcp-peer-v27.inc b/drivers/gpu/drm/apple/dcp-peer-v27.inc new file mode 100644 index 0000000000000..9bae0594b5843 --- /dev/null +++ b/drivers/gpu/drm/apple/dcp-peer-v27.inc @@ -0,0 +1,154 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ + +/* Bring-up gate until the framebuffer merge lifecycle is implemented. This + * only reserves an otherwise unrouted DCP; it never replaces a physical route. + */ +static bool peer_allocation_probe; +module_param(peer_allocation_probe, bool, 0444); +MODULE_PARM_DESC(peer_allocation_probe, "Test firmware27 logical peer allocation (merge bring-up)"); + +bool dcp_peer_tiling_state_v27(struct apple_dcp *dcp, bool write, + u32 event, u32 param, u32 *value) +{ + struct apple_dcp *peer; + + guard(mutex)(&dcp_peer_lock); + if (!peer_allocation_probe || dcp->peer_unbinding || dcp->crashed) + return false; + peer = dcp->peer_reserved ?: dcp->peer_owner; + if (!peer || peer->peer_unbinding || peer->crashed) + return false; + /* A resource reservation is not a framebuffer pair. Before the + * split-mode preparation, keep the established untiled callback path; + * zero-initialized fields are not yet valid tiling-manager state. + */ + if (!dcp->peer_tiling.vftg || !peer->peer_tiling.vftg) + return false; + if (write) + return dcp_peer_tiling_set_v27(&dcp->peer_tiling, + &peer->peer_tiling, event, param, *value); + return dcp_peer_tiling_get_v27(&dcp->peer_tiling, &peer->peer_tiling, + event, param, value); +} + +void dcp_peer_display_ids_v27(struct apple_dcp *dcp, u32 own, u32 related) +{ + struct apple_dcp *peer; + + guard(mutex)(&dcp_peer_lock); + peer = dcp->peer_reserved ?: dcp->peer_owner; + if (!peer || own == related || dcp->peer_unbinding || peer->peer_unbinding) + return; + /* D575 carries external display IDs, not crossbar endpoint indices. */ + dcp->peer_tiling.display_id = own; + dcp->peer_tiling.display_id_valid = true; + peer->peer_tiling.display_id = related; + peer->peer_tiling.display_id_valid = true; +} + +static void dcp_peer_release_locked(struct apple_dcp *dcp) +{ + if (dcp->peer_reserved) { + /* A link-allocation release is not a DMA-stop acknowledgement. + * Keep a prepared pair owned until the AP has powered both down. + * Firmware may still need the tiling rendezvous state to stop. + */ + if (dcp->peer_tiling.vftg || dcp->peer_tiling.member || + dcp->peer_reserved->peer_tiling.vftg || + dcp->peer_reserved->peer_tiling.member) { + dcp->peer_tiling.modeset_complete = false; + dcp->peer_reserved->peer_tiling.modeset_complete = false; + return; + } + memset(&dcp->peer_tiling, 0, sizeof(dcp->peer_tiling)); + memset(&dcp->peer_reserved->peer_tiling, 0, + sizeof(dcp->peer_reserved->peer_tiling)); + dcp->peer_reserved->peer_owner = NULL; + dcp->peer_reserved = NULL; + } +} + +int dcp_peer_hints_v27(struct apple_dcp *dcp, struct dptx_port *port, + const void *data, size_t size, void *reply, size_t reply_size) +{ + struct dcp_display_hints_v27 *r = reply; + struct apple_dcp *peer = NULL; + struct drm_crtc *crtc; + u32 extra; + + if (size != sizeof(*r) || reply_size != sizeof(*r)) + return -EINVAL; + memcpy(reply, data, sizeof(*r)); + /* The first word is hint flags, not a return code. The completion + * status has a field of its own, leaving the request hints unchanged. + */ + r->status = 0; + if (!peer_allocation_probe) + return 0; + if (!of_device_is_compatible(dcp->dev->of_node, "apple,t6050-dcpext") || + port->unit || !dcp->crtc) + return 0; + + extra = dcp_peer_extra_pipes(r->hints, &port->caps); + memset(&r->primary_id, 0, 32); + guard(mutex)(&dcp_peer_lock); + if (dcp->peer_unbinding || dcp->crashed) + return -ENODEV; + if (!extra || !(r->hints[0] & 1)) { + dcp_peer_release_locked(dcp); + memset(&r->primary_id, 0, 32); + return 0; + } + /* This candidate implements a non-tiled, two-pipe reservation. Tiled + * panels and more than one peer require the full display allocator. + */ + if (extra != 1 || (r->hints[0] & 2) || dcp->peer_owner) + return 0; + peer = dcp->peer_reserved; + if (!peer) { + drm_for_each_crtc(crtc, dcp->crtc->base.dev) { + struct apple_dcp *p = platform_get_drvdata(to_apple_crtc(crtc)->dcp); + + /* A dedicated, unrouted peer cannot race Type-C's allocator + * or a fixed connector's HPD handler in this bring-up step. + */ + if (!p || !p->active || p->crashed || p->peer_unbinding || + p->peer_owner || p->peer_reserved || p->fixed_phy || + p->nr_typec_routes || p->active_typec_route || + (p->connector && p->connector->connected) || + !dcp_peer_index_allowed(dcp->index, p->index) || + p->dptx_die != dcp->dptx_die || + !p->crtc || p->crtc->base.dev != dcp->crtc->base.dev || + p->fw_compat != DCP_FIRMWARE_V_27_0 || + !p->dptxport[0].enabled || + p->dptxport[0].caps.width < port->caps.width || + p->dptxport[0].caps.total_rate < port->caps.total_rate || + p->dptxport[0].caps.active_rate < port->caps.active_rate || + p->dptxport[0].connected || p->dptxport[1].connected || + !of_device_is_compatible(p->dev->of_node, "apple,t6050-dcpext")) + continue; + if (!peer || p->index < peer->index) + peer = p; + } + if (!peer) { + dev_info(dcp->dev, "no eligible idle peer for %u extra pipe\n", extra); + return 0; + } + dcp->peer_reserved = peer; + peer->peer_owner = dcp; + dev_info(dcp->dev, "reserved peer DCP%u for %ux%u, total %u active %u\n", + peer->index, get_unaligned_le32(r->hints + 0x38), + get_unaligned_le32(r->hints + 0x3c), + get_unaligned_le32(r->hints + 0x40), + get_unaligned_le32(r->hints + 0x44)); + } + /* The display allocation update writes primary/peer endpoint IDs. + * The extra pipe count supplies byte8. This is full dual-pipe, so the + * T6050 limited-dual-pipe flag (byte9 bit1) remains clear. + */ + r->primary_id = cpu_to_le32(dcp->index); + r->peer_id = cpu_to_le32(peer->index); + r->extra_pipes = extra; + r->flags &= ~3; + return 0; +} diff --git a/drivers/gpu/drm/apple/dcp-peer-video-probe-v27.inc b/drivers/gpu/drm/apple/dcp-peer-video-probe-v27.inc new file mode 100644 index 0000000000000..24c985a91195f --- /dev/null +++ b/drivers/gpu/drm/apple/dcp-peer-video-probe-v27.inc @@ -0,0 +1,250 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* Opt-in A364/A365 bring-up. Installs derived modes without selecting them; + * paired power, modeset and swap ownership must be implemented separately. + */ +#include "iomfb-peer-v27.h" + +struct dcp_peer_video_wait { + struct kref refs; + struct completion done; + size_t size; + u8 reply[sizeof(struct dcp_peer_get_video_resp_v27)]; +}; + +static void dcp_peer_video_free(struct kref *ref) +{ + kfree(container_of(ref, struct dcp_peer_video_wait, refs)); +} + +static void dcp_peer_video_done(struct apple_dcp *dcp, void *out, void *cookie) +{ + struct dcp_peer_video_wait *wait = cookie; + + if (wait->size) + memcpy(wait->reply, out, wait->size); + complete(&wait->done); + kref_put(&wait->refs, dcp_peer_video_free); +} + +static int dcp_peer_video_call_stream(struct apple_dcp *dcp, bool independent, + const char *tag, + void *in, size_t in_size, void *out, size_t out_size) +{ + struct dcp_method_entry method = { .name = "peer_video_test" }; + struct dcp_peer_video_wait *wait; + int ret = 0; + + if (out_size > sizeof(wait->reply)) + return -EINVAL; + /* An unrelated AP thread gets its own firmware stream. A369/A367/etc. + * must not be mistaken for nested callbacks of the pending A413. + * Stream1 (the driver's OOB context) is independent of stream0. + */ + if (independent && (dcp->ch_oobcmd.depth || dcp->ch_oobcb.depth)) + return -EBUSY; + wait = kzalloc(sizeof(*wait), GFP_KERNEL); + if (!wait) + return -ENOMEM; + memcpy(method.tag, tag, sizeof(method.tag)); + wait->size = out_size; + init_completion(&wait->done); + kref_init(&wait->refs); + kref_get(&wait->refs); + dcp_push(dcp, independent, &method, in_size, out_size, in, + dcp_peer_video_done, wait); + if (!wait_for_completion_timeout(&wait->done, + msecs_to_jiffies(!memcmp(tag, "A413", 4) ? 8500 : 2000))) + ret = -ETIMEDOUT; + else if (out_size) + memcpy(out, wait->reply, out_size); + /* A late ACK owns only the heap cookie, never a caller's stack. */ + kref_put(&wait->refs, dcp_peer_video_free); + return ret; +} + +static int dcp_peer_video_call(struct apple_dcp *dcp, const char *tag, + void *in, size_t in_size, void *out, size_t out_size) +{ + return dcp_peer_video_call_stream(dcp, false, tag, in, in_size, out, out_size); +} + +#include "dcp-peer-modeset-probe-v27.inc" + +static bool dcp_peer_video_idle(struct apple_dcp *dcp) +{ + return dcp->active && !dcp->crashed && !dcp->peer_unbinding && + !READ_ONCE(dcp->mode_state.changing) && !dcp->ch_cmd.depth && !dcp->ch_cb.depth && + !dcp->ch_async.depth && !dcp->ch_oobcmd.depth && + !dcp->ch_oobcb.depth && !dcp->ch_oobasync.depth; +} + +/* Shared by the explicit diagnostic and normal DRM mode selection. + * Install4 chooses a pair only when firmware's A364 requires one. + */ +static int dcp_peer_install_video_v27(struct apple_dcp *dcp, struct apple_dcp *peer, + u32 timing, u32 color, u32 install) +{ + struct device *dev = dcp->dev; + struct dcp_peer_get_video_req_v27 req = {}; + struct dcp_peer_get_video_resp_v27 resp; + struct dcp_peer_insert_video_req_v27 insert = {}; + struct dcp_peer_insert_video_resp_v27 inserted; + __le32 enabled = cpu_to_le32(1); + int ret; + + req.timing_id = cpu_to_le32(timing); + req.color_id = cpu_to_le32(color); + ret = dcp_peer_video_call(dcp, "A364", &req, sizeof(req), &resp, sizeof(resp)); + if (ret) + return ret; + dev_info(dev, "peer video timing=%u color=%u success=%u merge=%u timing=%*ph color=%*ph\n", + timing, color, resp.success, resp.requires_merge, + 64, resp.video.timing, 32, resp.video.color); + dev_info(dev, "peer timing tail=%*ph\n", 16, resp.video.timing + 64); + if (!(resp.success & 1)) { + ret = -ENOENT; + return ret; + } + if (!install) + return ret; + /* Mode3 explicitly tests a reserved pair using an existing lower-rate + * timing. It does not advertise a mode, raise a limit, or add scanout. + * Keep this diagnostic independent of production mode selection. + */ + if (!(resp.requires_merge & 1) && install != 3) { + if (install == 4 && (dcp->peer_tiling.vftg || peer->peer_tiling.vftg)) { + u8 power[16] = {1}, response[8]; + + ret = dcp_peer_probe_quiesce(dcp, peer); + if (ret) + return ret; + /* A mode-only DRM commit does not call CRTC poweron again. */ + ret = dcp_peer_video_call(dcp, "A484", power, sizeof(power), + response, sizeof(response)); + if (ret || get_unaligned_le32(response + 4)) + return ret ?: -EIO; + } + return -EOPNOTSUPP; + } + ret = dcp_peer_probe_quiesce(dcp, peer); + if (ret) + return ret; + insert.video = resp.video; + dcp_peer_split_timing_v27(&insert.video); + insert.timing_id = cpu_to_le32(timing + 10000); + insert.color_id = cpu_to_le32(color + 10000); + ret = dcp_peer_probe_prepare_vftg(dcp, peer, &insert); + if (ret) + return ret; + ret = dcp_peer_video_call(peer, "A366", &enabled, sizeof(enabled), NULL, 0); + if (ret) + return ret; + ret = dcp_peer_video_call(dcp, "A365", &insert, sizeof(insert), + &inserted, sizeof(inserted)); + if (!ret && !(inserted.success & 1)) + ret = -EIO; + if (ret) + return ret; + /* The first call's in/out timing and color go to the peer. */ + insert.video = inserted.video; + ret = dcp_peer_video_call(peer, "A365", &insert, sizeof(insert), + &inserted, sizeof(inserted)); + if (!ret && !(inserted.success & 1)) + ret = -EIO; + dev_info(dev, "peer derived mode timing=%u color=%u width=%u height=%u clock=%u result=%d (not selected)\n", + timing + 10000, color + 10000, + get_unaligned_le32(insert.video.timing + 8), + get_unaligned_le32(insert.video.timing + 0x28), + get_unaligned_le32(insert.video.timing + 0x44), ret); + if (!ret && install >= 2) + ret = dcp_peer_probe_modeset(dcp, peer, &insert, + resp.requires_merge & 1); + return ret; +} + +static ssize_t peer_video_test_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct apple_dcp *dcp, *peer; + struct device *peer_dev = NULL; + u32 timing, color, install; + char extra; + int ret; + + if (sscanf(buf, "%u %u %u %c", &timing, &color, &install, &extra) != 3 || + install > 3 || timing > U32_MAX - 10000 || color > U32_MAX - 10000) + return -EINVAL; + if (!peer_allocation_probe) + return -EOPNOTSUPP; + /* Trylocks avoid the sysfs-removal/device-lock inversion. Hold both + * devices across RPC waits so neither devm-backed DCP can be unbound. + */ + if (!device_trylock(dev)) + return -EBUSY; + dcp = dev_get_drvdata(dev); + { + guard(mutex)(&dcp_peer_lock); + peer = dcp ? dcp->peer_reserved : NULL; + if (!peer || peer->peer_owner != dcp || !dcp_peer_video_idle(dcp) || + !dcp_peer_video_idle(peer) || !dcp->crtc || !peer->crtc || + !dcp->crtc->base.state || !peer->crtc->base.state || + (install && (dcp->crtc->base.state->active || + peer->crtc->base.state->active)) || peer->fixed_phy || + peer->nr_typec_routes || peer->peer_tiling.member || + dcp->peer_tiling.member || + dcp->fw_compat != DCP_FIRMWARE_V_27_0 || + peer->fw_compat != DCP_FIRMWARE_V_27_0 || + (peer->connector && peer->connector->connected)) { + ret = -EBUSY; + goto unlock; + } + if (!device_trylock(peer->dev)) { + ret = -EBUSY; + goto unlock; + } + peer_dev = peer->dev; + } + ret = dcp_peer_install_video_v27(dcp, peer, timing, color, install); +unlock: + if (peer_dev) + device_unlock(peer_dev); + device_unlock(dev); + return ret ? ret : count; +} +static DEVICE_ATTR_WO(peer_video_test); + +static ssize_t peer_link_test_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct apple_dcp *dcp; + bool enabled; + int ret; + + if (kstrtobool(buf, &enabled)) + return -EINVAL; + if (!peer_allocation_probe || !device_trylock(dev)) + return -EBUSY; + dcp = dev_get_drvdata(dev); + if (!dcp || !dcp->active || dcp->crashed || !dcp->fixed_phy || + dcp->nr_typec_routes || dcp->connector_type != DRM_MODE_CONNECTOR_HDMIA) { + ret = -EOPNOTSUPP; + goto unlock; + } + { + guard(mutex)(&dcp_peer_lock); + if (dcp->peer_tiling.vftg || dcp->peer_tiling.member) { + ret = -EBUSY; + goto unlock; + } + } + /* Use the existing disconnect/connect lifecycle, serialized with HPD. + * This diagnostic models link re-enumeration, not a physical cable pull. + */ + ret = dcp_typec_test_link(to_platform_device(dev), enabled); +unlock: + device_unlock(dev); + return ret ? ret : count; +} +static DEVICE_ATTR_WO(peer_link_test); diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index caafa8a8fc739..24cc8c18c0809 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -68,6 +69,7 @@ #include "iomfb_internal.h" #include "iomfb_m4.h" #include "ibootep.h" +#include "iomfb_internal.h" #include "parser.h" #include "trace.h" @@ -117,9 +119,12 @@ static int dcp_pin_live_session(struct apple_dcp *dcp) return -ENOMEM; /* Keep the DMA device, domain usage and driver code valid until reset. */ + if (!try_module_get(THIS_MODULE)) { + kfree(session); + return -ENODEV; + } session->dev = get_device(dcp->dev); session->group = iommu_group_get(dcp->dev); - __module_get(THIS_MODULE); WRITE_ONCE(dcp->retain_dma, true); mutex_lock(&dcp_retained_sessions_lock); list_add_tail(&session->link, &dcp_retained_sessions); @@ -235,6 +240,9 @@ bool dcp_t6041_ext_enabled(void) #include "iomfb_v26_6_swap.h" #include "dcp-peer-v26_6.inc" +/* The firmware 27 pair shares dcp_peer_lock and dcp_peer_forget() above. */ +#include "dcp-peer-v27.inc" +#include "dcp-peer-probe-v27.inc" /* * The backlight maximum of a desktop 26.x panel whose node has no @@ -612,9 +620,10 @@ void dcp_drm_crtc_page_flip(struct apple_dcp *dcp, ktime_t now) } /* - * A desktop 26.x external processor does not hand out the sink's EDID through - * its DCPAV service, but exports the TimingElements and DisplayAttributes - * of every sink it drives, on HDMI and Type-C alike. + * A desktop 26.x external processor, and T6050's on firmware 27, does not + * hand out the sink's EDID through its DCPAV service, but exports the + * TimingElements and DisplayAttributes of every sink it drives, on HDMI and + * Type-C alike. */ static bool dcp_sink_metadata(struct apple_dcp *dcp) { @@ -1408,6 +1417,7 @@ static void dcp_rtk_crashed(void *cookie, const void *crashlog, size_t crashlog_ return; } dcp->crashed = true; + dcp_peer_forget(dcp); dev_err(dcp->dev, "DCP has crashed\n"); if (dcp_fail_stop(dcp) && dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) { @@ -1470,6 +1480,13 @@ static int dcp_rtk_shmem_setup(void *cookie, struct apple_rtkit_shmem *bfr) phy_addr = iommu_iova_to_phys(domain, bfr->iova); if (!phy_addr) return -ENOMEM; + /* Firmware 27's DVAs resolve through its own retained mappings. */ + if (dcp->fw_compat == DCP_FIRMWARE_V_27_0 && + !dcp_fw_phys_in_dram(phy_addr)) { + dev_err(dcp->dev, "shmem_setup: iova %pad is not in DRAM\n", + &bfr->iova); + return -EACCES; + } /* memremap() below needs the whole buffer physically contiguous */ ret = dcp_check_iova_contiguous(domain, bfr->iova, bfr->size, @@ -3889,6 +3906,7 @@ int dcp_start(struct platform_device *pdev) return iomfb_v26_6_start(dcp); if (dcp->fw_compat == DCP_FIRMWARE_V_26_6_DESKTOP) return dcp_start_desktop(dcp); + init_completion(&dcp->typec_display_ready); /* * The T6030 firmware session is adopted for the internal panel. @@ -3930,7 +3948,9 @@ int dcp_start(struct platform_device *pdev) ret); } - if (dcp->phy && dcp->fw_compat >= DCP_FIRMWARE_V_13_5) { + if ((dcp->phy || (dcp->fw_compat == DCP_FIRMWARE_V_27_0 && + dcp->hw.num_dptx_ports)) && + dcp->fw_compat >= DCP_FIRMWARE_V_13_5) { ret = ibootep_init(dcp); if (ret) dev_warn(dcp->dev, "Failed to start IBOOT endpoint: %d\n", @@ -4071,6 +4091,9 @@ static void _dcp_poweroff(struct apple_dcp *dcp) case DCP_FIRMWARE_H17P: iomfb_poweroff_h17p(dcp); break; + case DCP_FIRMWARE_V_27_0: + iomfb_poweroff_v27(dcp); + break; default: WARN_ONCE(true, "Unexpected firmware version: %u\n", dcp->fw_compat); break; @@ -4168,6 +4191,9 @@ static void __maybe_unused dcp_sleep(struct apple_dcp *dcp) case DCP_FIRMWARE_H17P: iomfb_sleep_h17p(dcp); break; + case DCP_FIRMWARE_V_27_0: + iomfb_sleep_v27(dcp); + break; default: WARN_ONCE(true, "Unexpected firmware version: %u\n", dcp->fw_compat); break; @@ -4193,7 +4219,7 @@ static bool dcp_uses_soft_dpms(struct apple_dcp *dcp) /* The 14.7 panel has its own known-level blank and restore hooks. */ return (!iomfb_v14_7_panel_nits(dcp) && dcp_backlight_active(dcp) && dcp_has_panel(dcp)) || - (dcp->fw_compat == DCP_FIRMWARE_H17P && + ((dcp->fw_compat == DCP_FIRMWARE_H17P || dcp->fw_compat == DCP_FIRMWARE_V_27_0) && dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G && dcp->connector_type == DRM_MODE_CONNECTOR_eDP); } @@ -4257,10 +4283,13 @@ void dcp_poweron(struct platform_device *pdev) } if (dcp_uses_soft_dpms(dcp)) { - if (dcp->fw_compat == DCP_FIRMWARE_H17P && + if ((dcp->fw_compat == DCP_FIRMWARE_H17P || dcp->fw_compat == DCP_FIRMWARE_V_27_0) && dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G && !READ_ONCE(dcp->pipe_enabled_h17p)) { - iomfb_poweron_h17p(dcp); + if (dcp->fw_compat == DCP_FIRMWARE_V_27_0) + iomfb_poweron_v27(dcp); + else + iomfb_poweron_h17p(dcp); if (!READ_ONCE(dcp->pipe_enabled_h17p)) { dev_err(dcp->dev, "initial display pipe enable failed\n"); WRITE_ONCE(dcp->crashed, true); @@ -4358,6 +4387,9 @@ void dcp_poweron(struct platform_device *pdev) case DCP_FIRMWARE_H17P: iomfb_poweron_h17p(dcp); break; + case DCP_FIRMWARE_V_27_0: + iomfb_poweron_v27(dcp); + break; default: WARN_ONCE(true, "Unexpected firmware version: %u\n", dcp->fw_compat); break; @@ -4897,6 +4929,23 @@ static int dcp_get_disp_regs(struct apple_dcp *dcp) } dcp->nr_disp_registers = count; + for (i = 0; i < count; i++) + dcp->disp_firmware_base[i] = dcp->disp_registers[i]->start; + if (of_property_present(dcp->dev->of_node, "apple,firmware-physical-regs")) { + if (of_property_count_u64_elems(dcp->dev->of_node, + "apple,firmware-physical-regs") != count) + return -EINVAL; + ret = of_property_read_u64_array(dcp->dev->of_node, + "apple,firmware-physical-regs", + dcp->disp_firmware_base, count); + if (ret) + return ret; + for (i = 0; i < count; i++) + if (!IS_ALIGNED(dcp->disp_firmware_base[i], 4) || + resource_size(dcp->disp_registers[i]) > + U64_MAX - dcp->disp_firmware_base[i]) + return -EINVAL; + } return 0; } @@ -4952,6 +5001,13 @@ static enum dcp_firmware_version dcp_check_firmware_version(struct device *dev) char fw_str[DCP_FW_VERSION_STR_LEN]; int ret; + if (hw->firmware_compat == DCP_FIRMWARE_V_27_0) { + ret = dcp_read_fw_version(dev, "apple,firmware-compat", compat_str); + if (ret < 0 || strcmp(compat_str, "27.0.0")) + return DCP_FIRMWARE_UNKNOWN; + return DCP_FIRMWARE_V_27_0; + } + /* * SoCs introduced with H17-generation firmware pin its interface; the * loader may not recognise their boot firmware version. @@ -5154,6 +5210,56 @@ VISIBLE_IF_KUNIT void dcp_enable_typec_work(struct apple_dcp *dcp) } } +static int dcp_firmware_metadata_v27(struct apple_dcp *dcp) +{ + struct device *dev = dcp->dev; + struct device_node *fb; + struct resource resource; + int clocks, index, ret; + + /* + * The framebuffer region is the one the boot loader scanned out of; it + * backs the firmware's default framebuffer. An external processor the + * boot loader drove no display from (T6050's dcpext4, the HDMI port's) + * has none: it binds without one, and its default-framebuffer requests + * are refused (dfb_region_size 0). + */ + index = of_property_match_string(dev->of_node, "memory-region-names", + "framebuffer"); + if (index == -ENODATA || index == -EINVAL) { + dcp->dfb_region_size = 0; + dev_info(dev, "no boot framebuffer\n"); + goto clocks; + } + if (index < 0) + return index; + fb = of_parse_phandle(dev->of_node, "memory-region", index); + if (!fb) + return -EINVAL; + ret = of_address_to_resource(fb, 0, &resource); + of_node_put(fb); + if (ret) + return ret; + dcp->dfb_region_size = resource_size(&resource); + if (!dcp->dfb_region_size) + return -EINVAL; + +clocks: + clocks = of_property_count_u32_elems(dev->of_node, "apple,clock-ids"); + if (clocks == -ENODATA && + of_property_present(dev->of_node, "apple,clock-ids")) + clocks = 0; + if (clocks < 0 || clocks > 64) + return -EINVAL; + ret = devm_clk_bulk_get_all(dev, &dcp->firmware_clocks); + if (ret < 0) + return ret; + if (ret != clocks) + return -EINVAL; + dcp->num_firmware_clocks = clocks; + return 0; +} + static bool dcp_external_is_idle(struct apple_dcp *dcp) { u32 control = readl_relaxed(dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL); @@ -5250,6 +5356,14 @@ static int dcp_comp_bind_pipe(struct device *dev, struct device *main, void *dat u32 cpu_ctrl; int ret; + if (dcp->fw_compat == DCP_FIRMWARE_V_27_0) { + INIT_LIST_HEAD(&dcp->iomfb_boolean_properties); + ret = devm_add_action_or_reset(dev, + iomfb_firmware_state_cleanup_v27, dcp); + if (ret) + return ret; + } + if (READ_ONCE(dcp->quiescing)) return dev_err_probe(dev, -EBUSY, "Live DCP session cannot be rebound; reboot required\n"); @@ -5282,7 +5396,8 @@ static int dcp_comp_bind_pipe(struct device *dev, struct device *main, void *dat return dev_err_probe(dev, ret, "Previous scanout has not been stopped\n"); - ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(42)); + ret = dma_set_mask_and_coherent(dev, + DMA_BIT_MASK(dcp->hw.firmware_dma_bits ?: 42)); if (ret) return ret; @@ -5419,12 +5534,18 @@ static int dcp_comp_bind_pipe(struct device *dev, struct device *main, void *dat goto err_piodma; } + if (dcp->fw_compat == DCP_FIRMWARE_V_27_0) { + ret = dcp_firmware_metadata_v27(dcp); + if (ret) + goto err_piodma; + } /* * T6041: the device tree carries no DCP clock, and the boot loader * left the coprocessor clocked; D408 then reports 0 Hz, as the * driver this instance was ported from did. */ - if (dcp->fw_compat == DCP_FIRMWARE_V_26_6_DESKTOP) + if (dcp->fw_compat == DCP_FIRMWARE_V_26_6_DESKTOP || + (dcp->fw_compat == DCP_FIRMWARE_V_27_0 && !dcp->num_firmware_clocks)) dcp->clk = devm_clk_get_optional(dev, NULL); else dcp->clk = devm_clk_get(dev, NULL); @@ -5526,8 +5647,24 @@ static int dcp_comp_bind_pipe(struct device *dev, struct device *main, void *dat static void dcp_ext_give_up(struct apple_dcp *dcp, int err, const char *when); /* - * The T8122 dcpext boots its firmware here. If that fails, it must not take - * the internal panel's DRM device down with it: keep it as an unusable pipe. + * The external processors whose failure must not take the DRM device down: + * T8122's, T6041's and T6050's dcpext. One that fails to probe or bind joins + * as an unusable pipe (ext_dead), and the other displays go on without it. + * With every processor failed the DRM device has no pipe and does not bind. + */ +static bool dcp_ext_is_failsafe(struct device_node *np) +{ + return of_device_is_compatible(np, "apple,t8122-dcpext") || + of_device_is_compatible(np, "apple,t6041-dcpext") || + of_device_is_compatible(np, "apple,t6050-dcpext"); +} + +/* + * A failsafe dcpext (dcp_ext_is_failsafe()) boots its firmware here. If that + * fails, it must not take the other displays' DRM device down with it: keep + * it as an unusable pipe. Its routes have registered by now and cannot be + * taken back, so a deferral here would be final as well: probe waits for the + * suppliers bind looks up (dcp_ext_bind_suppliers()) so that none comes here. */ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) { @@ -5555,6 +5692,7 @@ VISIBLE_IF_KUNIT void dcp_comp_unbind(struct device *dev, struct device *main, v if (!dcp) return; + dcp_peer_forget(dcp); /* * Stop the native initializer before looking up its owner, including * the dead-session path. Owner allocation may have landed in an open @@ -5961,8 +6099,7 @@ static int dcp_platform_probe_pipe(struct platform_device *pdev) dcp->desktop->client = GENMASK_ULL(63, 48) | (get_random_u64() & GENMASK_ULL(47, 4)); } - dcp->ext_failsafe = of_device_is_compatible(dev->of_node, "apple,t8122-dcpext") || - of_device_is_compatible(dev->of_node, "apple,t6041-dcpext"); + dcp->ext_failsafe = dcp_ext_is_failsafe(dev->of_node); dcp->external = of_device_is_compatible(dev->of_node, "apple,t6030-dcpext"); dcp->desktop_external = of_device_is_compatible(dev->of_node, "apple,t6041-dcpext"); dcp->external_native = dcp->external && dcp_t6030_ext_native(dev->of_node); @@ -6347,7 +6484,7 @@ static int dcp_platform_probe_pipe(struct platform_device *pdev) static void dcp_ext_give_up(struct apple_dcp *dcp, int err, const char *when) { WRITE_ONCE(dcp->ext_dead, true); - dev_err(dcp->dev, "external display processor unusable (%s failed: %d); the internal display goes on without it\n", + dev_err(dcp->dev, "external display processor unusable (%s failed: %d); the other displays go on without it\n", when, err); dcp_typec_routes_disable(dcp); } @@ -6390,6 +6527,46 @@ static int dcp_ext_route_suppliers(struct device *dev) return 0; } +/* -EPROBE_DEFER while the device of available node @np has not bound. */ +static int dcp_ext_supplier_bound(struct device_node *np) +{ + struct platform_device *pdev; + int ret = 0; + + if (!np || !of_device_is_available(np)) + return 0; + pdev = of_find_device_by_node(np); + if (!pdev || !device_is_bound(&pdev->dev)) + ret = -EPROBE_DEFER; + if (pdev) + put_device(&pdev->dev); + return ret; +} + +/* + * Whether the suppliers that component bind looks up, and that probe does not + * otherwise wait for, are there: the PIODMA child's DART (a child node's + * supplier does not hold up the parent's probe) and the mailbox. Bind runs + * after the Type-C routes registered, where a deferral can only leave the + * dcpext unusable for the boot (dcp_comp_bind()). Returns -EPROBE_DEFER until + * they have bound; a missing one is left to bind, which reports it. + */ +static int dcp_ext_bind_suppliers(struct device *dev) +{ + struct device_node *piodma __free(device_node) = + of_get_child_by_name(dev->of_node, "piodma"); + struct device_node *dart __free(device_node) = + piodma ? of_parse_phandle(piodma, "iommus", 0) : NULL; + struct device_node *mbox __free(device_node) = + of_parse_phandle(dev->of_node, "mboxes", 0); + int ret; + + ret = dcp_ext_supplier_bound(dart); + if (!ret) + ret = dcp_ext_supplier_bound(mbox); + return ret; +} + /* * A T8132 dcpext's Type-C routes are only enabled for one the boot * loader handed over (see the display gate), and its USB-C port controllers @@ -6418,10 +6595,11 @@ static int dcp_platform_probe_t8132_ext(struct platform_device *pdev) } /* - * The T8122 dcpext is a component of the panel's DRM device, so it must - * always register one, or the panel stays on the boot framebuffer. Keep - * deferring while its suppliers may still appear (bounded by the deferred - * probe timeout); on any other failure join as an unusable pipe. + * A failsafe dcpext (dcp_ext_is_failsafe()) is a component of the other + * displays' DRM device, so it must always register one, or they stay on the + * boot framebuffer. Keep deferring while its suppliers, its routes' and its + * bind's, may still appear (bounded by the deferred probe timeout); on any + * other failure join as an unusable pipe. */ static int dcp_platform_probe(struct platform_device *pdev) { @@ -6439,11 +6617,12 @@ static int dcp_platform_probe(struct platform_device *pdev) if (of_device_is_compatible(dev->of_node, "apple,t8132-dcpext")) return dcp_platform_probe_t8132_ext(pdev); - if (!of_device_is_compatible(dev->of_node, "apple,t8122-dcpext") && - !of_device_is_compatible(dev->of_node, "apple,t6041-dcpext")) + if (!dcp_ext_is_failsafe(dev->of_node)) return dcp_platform_probe_pipe(pdev); ret = dcp_ext_route_suppliers(dev); + if (!ret) + ret = dcp_ext_bind_suppliers(dev); if (ret) goto unusable; @@ -6995,9 +7174,26 @@ static const struct apple_dcp_hw_data apple_dcp_hw_t6041_dcpext = { .sink_metadata_edid = true, }; +static const struct apple_dcp_hw_data apple_dcp_hw_t6050 = { + .firmware_dma_bits = 41, + .num_dptx_ports = 0, + .adopt_live_session = true, + .firmware_compat = DCP_FIRMWARE_V_27_0, +}; + +static const struct apple_dcp_hw_data apple_dcp_hw_t6050_dcpext = { + .firmware_dma_bits = 41, + .num_dptx_ports = 2, + .adopt_live_session = true, + .firmware_compat = DCP_FIRMWARE_V_27_0, + .sink_metadata_edid = true, +}; + static const struct of_device_id of_match[] = { { .compatible = "apple,t6041-dcp", .data = &apple_dcp_hw_t6041, }, { .compatible = "apple,t6041-dcpext", .data = &apple_dcp_hw_t6041_dcpext, }, + { .compatible = "apple,t6050-dcp", .data = &apple_dcp_hw_t6050 }, + { .compatible = "apple,t6050-dcpext", .data = &apple_dcp_hw_t6050_dcpext }, { .compatible = "apple,t6020-dcp", .data = &apple_dcp_hw_t6020, }, { .compatible = "apple,t6020-dcpext", .data = &apple_dcp_hw_t6020_dcpext, }, { .compatible = "apple,t8112-dcp", .data = &apple_dcp_hw_t8112, }, @@ -7027,6 +7223,7 @@ static struct platform_driver apple_platform_driver = { .shutdown = dcp_platform_shutdown, .driver = { .name = "apple-dcp", + .dev_groups = dcp_peer_probe_groups, .of_match_table = of_match, .pm = pm_sleep_ptr(&dcp_platform_pm_ops), }, diff --git a/drivers/gpu/drm/apple/dcp.h b/drivers/gpu/drm/apple/dcp.h index 14563a26602ce..dd6665c8c283f 100644 --- a/drivers/gpu/drm/apple/dcp.h +++ b/drivers/gpu/drm/apple/dcp.h @@ -58,6 +58,19 @@ void dcp_poweroff(struct platform_device *pdev); int dcp_typec_prepare_route(struct apple_dcp *dcp); void dcp_poweron(struct platform_device *pdev); void dcp_external_recover(struct platform_device *pdev); +int dcp_peer_hints_v27(struct apple_dcp *dcp, struct dptx_port *port, + const void *data, size_t size, void *reply, size_t reply_size); +bool dcp_peer_tiling_state_v27(struct apple_dcp *dcp, bool write, + u32 event, u32 param, u32 *value); +void dcp_peer_display_ids_v27(struct apple_dcp *dcp, u32 own, u32 related); +int dcp_peer_modeset_v27(struct apple_dcp *dcp, u32 timing, u32 color, bool vrr); +int dcp_peer_color_v27(struct apple_dcp *dcp, struct iomfb_set_matrix_req *matrix); +int dcp_peer_stop_v27(struct apple_dcp *dcp); +struct apple_dcp *dcp_peer_scanout_v27(struct apple_dcp *dcp); +void dcp_peer_swap_armed_v27(struct apple_dcp *dcp, u32 id); +void dcp_peer_swap_failed_v27(struct apple_dcp *dcp); +bool dcp_peer_swap_complete_v27(struct apple_dcp *dcp, u32 id, ktime_t now, + void (*report)(struct apple_dcp *, u32, ktime_t)); int dcp_set_crc(struct drm_crtc *crtc, bool enabled); int dcp_crtc_atomic_check(struct drm_crtc *crtc, struct drm_atomic_commit *commit); int dcp_get_connector_type(struct platform_device *pdev); diff --git a/drivers/gpu/drm/apple/dcp_backlight.c b/drivers/gpu/drm/apple/dcp_backlight.c index b151887274a26..00f29157241f8 100644 --- a/drivers/gpu/drm/apple/dcp_backlight.c +++ b/drivers/gpu/drm/apple/dcp_backlight.c @@ -96,7 +96,7 @@ static u32 brightness_part2[] = { static bool dcp_backlight_presents_nits(struct apple_dcp *dcp) { - return (dcp->fw_compat == DCP_FIRMWARE_H17P && + return ((dcp->fw_compat == DCP_FIRMWARE_H17P || dcp->fw_compat == DCP_FIRMWARE_V_27_0) && dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) || iomfb_v14_7_panel_nits(dcp); } @@ -274,7 +274,7 @@ int dcp_backlight_configure(struct apple_dcp *dcp, u32 maximum, unsigned long flags; int ret; - if ((dcp->fw_compat != DCP_FIRMWARE_H17P && + if ((dcp->fw_compat != DCP_FIRMWARE_H17P && dcp->fw_compat != DCP_FIRMWARE_V_27_0 && !iomfb_is_m4(dcp) && !iomfb_v14_7_panel_nits(dcp)) || !dcp_has_panel(dcp) || !dcp->crtc || !kick) return -EINVAL; diff --git a/drivers/gpu/drm/apple/dcpext_scanout.c b/drivers/gpu/drm/apple/dcpext_scanout.c index f3266931bb456..e652b8ac207a0 100644 --- a/drivers/gpu/drm/apple/dcpext_scanout.c +++ b/drivers/gpu/drm/apple/dcpext_scanout.c @@ -693,7 +693,10 @@ static ssize_t dcpext_pattern_store(struct device *dev, struct device_attribute } mutex_unlock(&dcp->hpd_mutex); /* Firmware DMA and callbacks may outlive any error returned by this worker. */ - __module_get(THIS_MODULE); + if (!try_module_get(THIS_MODULE)) { + atomic_set(&scanout->requested, 0); + return -ENODEV; + } queue_work(system_dfl_wq, &scanout->work); return count; } diff --git a/drivers/gpu/drm/apple/dptx-drive-v27.inc b/drivers/gpu/drm/apple/dptx-drive-v27.inc new file mode 100644 index 0000000000000..9b8c3e6af1fa2 --- /dev/null +++ b/drivers/gpu/drm/apple/dptx-drive-v27.inc @@ -0,0 +1,79 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright 2026 Aurora Silicon */ +struct dptxport_drive_v27 { + __le32 status_or_count; + u8 reserved0[12]; + __le32 count_or_status; + u8 reserved1[12]; + struct { + __le32 type; + __le32 voltage; + __le32 pre; + } lane[]; +} __packed; + +static int dptxport_call_drive_v27(struct apple_epic_service *service, bool set, + const void *data, size_t data_size, + void *reply_, size_t reply_size) +{ + const struct dptxport_drive_v27 *request = data; + struct dptxport_drive_v27 *reply = reply_; + struct dptx_port *dptx = service->cookie; + union phy_configure_opts opts = {}; + u32 count; + size_t size; + int ret; + + if (data_size < sizeof(*request) || reply_size < sizeof(*reply)) + return -EINVAL; + count = le32_to_cpu(set ? request->count_or_status : request->status_or_count); + if (count > 4 || count > dptx->lane_count) + return -EINVAL; + size = sizeof(*request) + count * sizeof(request->lane[0]); + if ((set && data_size < size) || (!set && reply_size < size)) + return -EINVAL; + + if (set) { + for (u32 i = 0; i < count; i++) { + + if (le32_to_cpu(request->lane[i].type) || + le32_to_cpu(request->lane[i].voltage) > 3 || + le32_to_cpu(request->lane[i].pre) > 3 || + le32_to_cpu(request->lane[i].voltage) + + le32_to_cpu(request->lane[i].pre) > 3) + return -EINVAL; + opts.dp.voltage[i] = le32_to_cpu(request->lane[i].voltage); + opts.dp.pre[i] = le32_to_cpu(request->lane[i].pre); + } + opts.dp.lanes = count; + opts.dp.link_rate = dptx->phy_ops.dp.link_rate; + opts.dp.set_voltages = 1; + if (count && !service->ep->dcp->dptx_tunnel) { + if (!dptx->atcphy) + return -ENODEV; + ret = phy_configure(dptx->atcphy, &opts); + if (ret) + return ret; + } + for (u32 i = 0; i < count; i++) { + dptx->drive_voltage[i] = opts.dp.voltage[i]; + dptx->drive_pre[i] = opts.dp.pre[i]; + } + } + + memset(reply, 0, reply_size); + if (set) { + reply->count_or_status = cpu_to_le32(count); + } else { + + reply->status_or_count = cpu_to_le32(count); + } + + if (reply_size >= size) { + for (u32 i = 0; i < count; i++) { + reply->lane[i].voltage = cpu_to_le32(dptx->drive_voltage[i]); + reply->lane[i].pre = cpu_to_le32(dptx->drive_pre[i]); + } + } + return 0; +} diff --git a/drivers/gpu/drm/apple/dptx-service.h b/drivers/gpu/drm/apple/dptx-service.h index 3d990a9ca1ea1..2cb8f68210eb8 100644 --- a/drivers/gpu/drm/apple/dptx-service.h +++ b/drivers/gpu/drm/apple/dptx-service.h @@ -5,14 +5,18 @@ #include "afk.h" #include "dptxep.h" +/* + * T8140 external firmware names its one port by the DCP index; the others, + * firmware 27 among them, by port unit 0 or 1. + */ static inline int dptx_service_port(const char *name, const char *class, - s64 unit, bool wire_v2, u32 index, + s64 unit, bool unit_is_index, u32 index, u32 num_ports) { if (!name || !class || strcmp(name, "dcpdptx-port-epic") || strcmp(class, "AppleDCPDPTXRemotePort")) return -EINVAL; - if (wire_v2) + if (unit_is_index) return num_ports == 1 && unit == index ? 0 : -EINVAL; return unit >= 0 && unit < 2 ? unit : -EINVAL; } diff --git a/drivers/gpu/drm/apple/dptxep.c b/drivers/gpu/drm/apple/dptxep.c index b2426f0a0506a..54464ddf1e9de 100644 --- a/drivers/gpu/drm/apple/dptxep.c +++ b/drivers/gpu/drm/apple/dptxep.c @@ -87,6 +87,8 @@ struct dptxport_apcall_set_tiled { __le32 retcode; }; +#include "dptx-drive-v27.inc" + /* * SetTiledDisplayHints payload, as DCP sends it for an LG UltraFine 5K * (offsets in bytes): tile location (column, row) at 0x08/0x0c, tiles @@ -275,7 +277,7 @@ dptxport_call_get_max_drive_settings(struct apple_epic_service *service, return -EINVAL; reply->retcode = cpu_to_le32(0); - if (service->ep->wire_v2) { + if (dcp_is_external_h17p(service->ep->dcp)) { memset(reply, 0, reply_size); put_unaligned_le32(3, reply_ + 20); put_unaligned_le32(3, reply_ + 24); @@ -284,6 +286,11 @@ dptxport_call_get_max_drive_settings(struct apple_epic_service *service, reply->max_drive_settings[0] = cpu_to_le32(0x3); reply->max_drive_settings[1] = cpu_to_le32(0x3); + if (service->ep->dcp->fw_compat == DCP_FIRMWARE_V_27_0) { + /* Protocol-27 drive settings: type, voltage, pre-emphasis. */ + reply->max_drive_settings[0] = cpu_to_le32(0); + memcpy(reply->_unk1, &reply->max_drive_settings[1], sizeof(__le32)); + } return 0; } @@ -424,7 +431,7 @@ static int dptxport_call_set_active_lane_count(struct apple_epic_service *servic u64 lane_count = le64_to_cpu(request->lane_count); - if (service->ep->wire_v2) { + if (dcp_is_external_h17p(dcp)) { ret = dptx_j700_validate_lanes(dptx, lane_count); if (ret) return ret; @@ -444,7 +451,8 @@ static int dptxport_call_set_active_lane_count(struct apple_epic_service *servic dptx->phy_ops.dp.lanes = lane_count; /* USB4 DP IN has no ATC PHY; still accept the lane count. */ dptx->phy_ops.dp.set_lanes = - service->ep->wire_v2 || dcp_is_usb4_output(dcp) || (dcp->dptx_phy > 3); + dcp_is_external_h17p(dcp) || dcp_is_usb4_output(dcp) || + (dcp->dptx_phy > 3); break; default: dev_err(dcp->dev, "set_active_lane_count: invalid lane count:%llu\n", lane_count); @@ -749,10 +757,14 @@ dptxport_call_activate(struct apple_epic_service *service, * No PHY mode change for a tunnel: the ATC PHY stays in USB4/TBT * mode for the whole connection. The DP IN adapter may only be woken * (DPTX_INACTIVE=0) from here; waking it earlier hangs the machine. - * Activate always replies success to DCP. + * Activate replies success to DCP, except that firmware 27 is told + * when the PHY could not switch to DisplayPort. */ - if (dptx->atcphy && !dcp->phy_managed_by_typec) - phy_set_mode_ext(dptx->atcphy, PHY_MODE_DP, dcp->index); + if (dptx->atcphy && !dcp->phy_managed_by_typec) { + ret = phy_set_mode_ext(dptx->atcphy, PHY_MODE_DP, dcp->index); + if (ret && dcp->fw_compat == DCP_FIRMWARE_V_27_0) + return ret; + } if (dcp->dptx_tunnel) { ret = dcp_tunnel_dpin_activate(dcp, dptx->unit, true); /* T6030 14.7: report a failed tunnel activation to DCP. */ @@ -924,6 +936,9 @@ static int dptxport_call(struct apple_epic_service *service, u32 idx, if (dptxport_dual_pipe_alloc(service, data, data_size, reply, reply_size)) return 0; + if (service->ep->dcp->fw_compat == DCP_FIRMWARE_V_27_0) + return dcp_peer_hints_v27(service->ep->dcp, dptx, data, + data_size, reply, reply_size); memcpy(reply, data, min(reply_size, data_size)); ret = dptxport_call_set_tiled_display_hint(service, data, data_size, reply, @@ -934,9 +949,15 @@ static int dptxport_call(struct apple_epic_service *service, u32 idx, return ret; } case DPTX_APCALL_GET_DRIVE_SETTINGS: + if (service->ep->dcp->fw_compat == DCP_FIRMWARE_V_27_0) + return dptxport_call_drive_v27(service, false, data, data_size, + reply, reply_size); return dptxport_call_get_drive_settings(service, data, data_size, reply, reply_size); case DPTX_APCALL_SET_DRIVE_SETTINGS: + if (service->ep->dcp->fw_compat == DCP_FIRMWARE_V_27_0) + return dptxport_call_drive_v27(service, true, data, data_size, + reply, reply_size); return dptxport_call_set_drive_settings(service, data, data_size, reply, reply_size); case DPTX_APCALL_ACTIVATE: @@ -1011,7 +1032,7 @@ static int dptxport_prepare(struct apple_epic_service *service, const char *name (!class || !class[0] || !strcmp(class, "AppleDCPDPTXRemotePort"))) port = unit >= 0 && unit < 2 ? unit : -EINVAL; else - port = dptx_service_port(name, class, unit, service->ep->wire_v2, + port = dptx_service_port(name, class, unit, dcp_is_external_h17p(dcp), dcp->index, dcp->hw.num_dptx_ports); if (port < 0) @@ -1020,6 +1041,21 @@ static int dptxport_prepare(struct apple_epic_service *service, const char *name return dptx_service_bind(service, &dcp->dptxport[port], unit); } +static void dptxport_init_props(struct apple_epic_service *service, s64 unit, + const void *props, size_t size) +{ + struct apple_dcp *dcp = service->ep->dcp; + struct dcp_parse_ctx ctx; + struct dcp_dptx_caps caps; + + if (unit < 0 || unit >= ARRAY_SIZE(dcp->dptxport) || + parse(props, size, &ctx) || parse_dptx_caps(&ctx, &caps)) + return; + dcp->dptxport[unit].caps = caps; + dev_info(dcp->dev, "DPTX%lld caps: width=%u total=%u active=%u\n", + unit, caps.width, caps.total_rate, caps.active_rate); +} + static void dptxport_init(struct apple_epic_service *service, const char *name, const char *class, s64 unit) { @@ -1093,8 +1129,12 @@ static int dptxport_group_call(struct apple_epic_service *service, u16 group, { struct dptx_port *dptx = service->cookie; - /* Keep native AFK-v2 training and its refusal policy distinct. */ - if (service->ep->wire_v2) + /* + * Keep the T8140 external (J700) training and its refusal policy + * distinct. It keys on that profile, not on the AFK v2 codec, which + * firmware 27 speaks with the common DPTX calls. + */ + if (dcp_is_external_h17p(service->ep->dcp)) return dptxport_v2_group_call(service, group, command, data, size, reply, room); @@ -1152,6 +1192,7 @@ static const struct apple_epic_service_ops dptxep_ops[] = { .prepare = dptxport_prepare, .teardown = dptxport_teardown, .init = dptxport_init, + .init_props = dptxport_init_props, .call = dptxport_call, }, { @@ -1306,7 +1347,8 @@ int dptx_controller_init(struct apple_dcp *dcp) int dptxep_init(struct apple_dcp *dcp) { struct apple_dcp_afkep *ep; - bool wire_v2 = dcp->hw.iomfb_method_profile == + bool wire_v2 = dcp->fw_compat == DCP_FIRMWARE_V_27_0 || + dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL; int ret; u32 port; diff --git a/drivers/gpu/drm/apple/dptxep.h b/drivers/gpu/drm/apple/dptxep.h index 211e72d9855ea..11cd860f9ee34 100644 --- a/drivers/gpu/drm/apple/dptxep.h +++ b/drivers/gpu/drm/apple/dptxep.h @@ -3,6 +3,7 @@ #include #include +#include "dcp-peer-v27.h" enum dptx_apcall { DPTX_APCALL_ACTIVATE = 0, @@ -49,6 +50,7 @@ enum dptx_link_rate { struct apple_epic_service; struct dptx_port { + struct dcp_dptx_caps caps; bool enabled, connected; /* A published session closed; unlike initial discovery, this is terminal. */ bool closed; @@ -63,6 +65,7 @@ struct dptx_port { u32 link_rate, pending_link_rate; u32 drive_settings[2]; u32 trained_voltage[4], trained_pre[4]; + u32 drive_voltage[4], drive_pre[4]; /* * The sink's tile topology as DCP reports it in SetTiledDisplayHints: * tiles across and down, and this stream's tile. Valid once diff --git a/drivers/gpu/drm/apple/epic/dpavservep.c b/drivers/gpu/drm/apple/epic/dpavservep.c index a92af3295fcf8..9621a86b42228 100644 --- a/drivers/gpu/drm/apple/epic/dpavservep.c +++ b/drivers/gpu/drm/apple/epic/dpavservep.c @@ -712,7 +712,8 @@ int dpavservep_init(struct apple_dcp *dcp) if (IS_ERR(ep)) return PTR_ERR(ep); /* External H17P uses AFKV2 service calls with host-owned OOB buffers. */ - ep->wire_v2 = dcp->hw.iomfb_method_profile == + ep->wire_v2 = dcp->fw_compat == DCP_FIRMWARE_V_27_0 || + dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL; ep->inline_v2 = false; dcp->dcpavservep = ep; diff --git a/drivers/gpu/drm/apple/iomfb-boolean-properties-v27.inc b/drivers/gpu/drm/apple/iomfb-boolean-properties-v27.inc new file mode 100644 index 0000000000000..b7b617b37994d --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb-boolean-properties-v27.inc @@ -0,0 +1,51 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright 2026 Aurora Silicon */ + +struct dcp_boolean_property_v27 { + struct list_head node; + char name[64]; + bool value; +}; + +static bool trampoline_set_boolean_property_v27(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + const u8 *request = in; + struct dcp_boolean_property_v27 *property; + char name[64]; + + trace_iomfb_callback(dcp, tag, __func__); + + memcpy(name, request, sizeof(name)); + name[sizeof(name) - 1] = '\0'; + put_unaligned_le32(0, out); + + list_for_each_entry(property, &dcp->iomfb_boolean_properties, node) + if (!strcmp(property->name, name)) + goto update; + if (dcp->iomfb_boolean_property_count >= 64) + return true; + property = kzalloc(sizeof(*property), GFP_KERNEL); + if (!property) + return true; + memcpy(property->name, name, sizeof(name)); + dcp->iomfb_boolean_property_count++; + list_add_tail(&property->node, &dcp->iomfb_boolean_properties); +update: + property->value = request[64] & 1; + put_unaligned_le32(1, out); + return true; +} + +void iomfb_firmware_state_cleanup_v27(void *data) +{ + struct apple_dcp *dcp = data; + struct dcp_boolean_property_v27 *property, *next; + + external_counts_cleanup_v27(dcp); + list_for_each_entry_safe(property, next, &dcp->iomfb_boolean_properties, node) { + list_del(&property->node); + kfree(property); + dcp->iomfb_boolean_property_count--; + } +} diff --git a/drivers/gpu/drm/apple/iomfb-dfb.h b/drivers/gpu/drm/apple/iomfb-dfb.h new file mode 100644 index 0000000000000..8175cb2d3c922 --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb-dfb.h @@ -0,0 +1,122 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright 2026 Aurora Silicon */ + +#ifndef __APPLE_IOMFB_DFB_H__ +#define __APPLE_IOMFB_DFB_H__ + +#include +#include +#include +#include +#include + +static inline u64 apple_dcp_dfb_round_size(u64 size) +{ + unsigned int bits = fls64(size); + u64 unit, half, tail; + + if (bits <= 24) + return size; + unit = 1ULL << (bits - 24); + half = unit >> 1; + tail = size & (unit - 1); + size &= ~(unit - 1); + if (tail > half || (tail == half && (size & unit))) + size += unit; + return size; +} + +static inline u64 apple_dcp_dfb_plane_size(u32 width, u32 height, u32 bpe, + u32 compression) +{ + u64 stride, data, cols, rows, tw = 16, th = 16; + unsigned int order; + + if (!compression) { + stride = ALIGN((u64)width * bpe, 128); + return ALIGN(apple_dcp_dfb_round_size(stride * height), 0x4000); + } + if (compression == 2 && bpe > 4) { + th = 8; + if (bpe == 16) + tw = 8; + } + cols = DIV_ROUND_UP(width, tw); + rows = DIV_ROUND_UP(height, th); + if (compression == 3) { + stride = tw * th * cols * bpe; + data = ALIGN(rows * stride, 128); + return data + 8 * roundup_pow_of_two(cols) * roundup_pow_of_two(rows); + } + order = ilog2(0x4000 / bpe); + stride = ALIGN((u64)width * bpe, (u64)bpe << (order / 2)); + data = ALIGN(ALIGN((u64)height, 1ULL << ((order + 1) / 2)) * stride, 128); + return data + ALIGN(2 * roundup_pow_of_two(cols) * roundup_pow_of_two(rows), 128); +} + +/* + * The DFB's surface format and bytes per element (the surface record's pix_size) + * for the firmware's format and compression; *alpha_plane for a separate alpha plane. + */ +static inline int apple_dcp_dfb_format(u32 format, u32 compression, + u32 *surface_format, u32 *bpe, + bool *alpha_plane) +{ + *alpha_plane = false; + switch (format) { + case 0x62336138: + if (compression == 2) { + format = 0x77343061; + *bpe = 8; + } else { + *bpe = 4; + *alpha_plane = true; + } + break; + case 0x77343061: + *bpe = 8; + break; + case 0x42475241: + case 0x77333072: + case 0x6c313072: + *bpe = 4; + break; + default: + return -EOPNOTSUPP; + } + *surface_format = format; + return 0; +} + +static inline int apple_dcp_dfb_allocation(u32 width, u32 height, u32 format, + u32 compression, u64 region_size, + u32 *surface_format, u32 *allocation) +{ + u32 bpe; + u64 size; + bool alpha_plane; + int ret; + + if (!width || !height || width > 16384 || height > 16384 || !region_size) + return -EINVAL; + if (compression != 0 && compression != 2 && compression != 3) + return -EOPNOTSUPP; + ret = apple_dcp_dfb_format(format, compression, &format, &bpe, &alpha_plane); + if (ret) + return ret; + size = apple_dcp_dfb_plane_size(width, height, bpe, compression); + if (alpha_plane) + size += apple_dcp_dfb_plane_size(width, height, 1, compression); + if (size > U32_MAX) + return -EOVERFLOW; + if (compression) + size = ALIGN(apple_dcp_dfb_round_size(size), 0x4000); + if (size > region_size) + return -ENOSPC; + if (!size || size > U32_MAX) + return -EOVERFLOW; + *surface_format = format; + *allocation = size; + return 0; +} +#endif diff --git a/drivers/gpu/drm/apple/iomfb-display-identity-v27.inc b/drivers/gpu/drm/apple/iomfb-display-identity-v27.inc new file mode 100644 index 0000000000000..1b3710c1c7627 --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb-display-identity-v27.inc @@ -0,0 +1,84 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright 2026 Aurora Silicon */ + +struct dcp_display_product_request_v27 { + u8 vendor[10]; + __le32 vendor_id; + __le32 vendor_length; + __le32 product_id; + u8 vendor_id_null, product_id_null; +} __packed; + +struct dcp_display_product_reply_v27 { + u8 vendor[10]; + __le32 vendor_id; + __le32 product_id; + u8 success, padding; +} __packed; + +static_assert(sizeof(struct dcp_display_product_request_v27) == 24); +static_assert(sizeof(struct dcp_display_product_reply_v27) == 20); +static_assert(offsetof(struct dcp_display_product_reply_v27, product_id) == 14); + +struct dcp_display_identity_callback_v27 { + void *output; + enum dcp_context_id context; +}; + +static void display_identity_done_v27(struct apple_dcp *dcp, void *out, void *cookie) +{ + const struct dcp_display_product_reply_v27 *reply = out; + struct dcp_display_identity_callback_v27 *cb = cookie; + bool vendor_matches = reply && (!reply->vendor[0] || + (!memcmp(reply->vendor, "APP", 3) && !reply->vendor[3])); + bool display_x1 = reply && (reply->success & 1) && vendor_matches && + (le32_to_cpu(reply->product_id) >> 1) == 0x5721; + + put_unaligned_le32(display_x1, cb->output); + dcp_ack(dcp, cb->context); + kfree(cb); +} + +static bool trampoline_display_identity_v27(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + static const struct dcp_method_entry method = { + .name = "display_product_id", .tag = {'A', '3', '7', '5'}, + }; + static const enum dcp_context_id contexts[] = { + DCP_CONTEXT_CB, DCP_CONTEXT_ASYNC, DCP_CONTEXT_OOBCB, DCP_CONTEXT_OOBASYNC, + }; + struct dcp_channel *channels[] = { + &dcp->ch_cb, &dcp->ch_async, &dcp->ch_oobcb, &dcp->ch_oobasync, + }; + struct dcp_display_product_request_v27 request = { + .vendor_length = cpu_to_le32(10), + }; + struct dcp_display_identity_callback_v27 *cb; + int context = -1; + + trace_iomfb_callback(dcp, tag, __func__); + + for (unsigned int i = 0; i < ARRAY_SIZE(channels); i++) { + struct dcp_channel *ch = channels[i]; + + if (ch->depth && ch->output[ch->depth - 1] == out) { + context = contexts[i]; + break; + } + } + if (WARN_ON(context < 0)) + return true; + cb = kzalloc(sizeof(*cb), GFP_KERNEL); + if (!cb) + return true; + cb->output = out; + cb->context = context; + if (dcp_push_nested(dcp, context, &method, sizeof(request), + sizeof(struct dcp_display_product_reply_v27), &request, + display_identity_done_v27, cb)) { + kfree(cb); + return true; + } + return false; +} diff --git a/drivers/gpu/drm/apple/iomfb-external-counts-v27.inc b/drivers/gpu/drm/apple/iomfb-external-counts-v27.inc new file mode 100644 index 0000000000000..ab5602821d3ca --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb-external-counts-v27.inc @@ -0,0 +1,90 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright 2026 Aurora Silicon */ + +static DEFINE_SPINLOCK(external_counts_lock_v27); +static u32 external_pipe_count_v27; +static u32 external_apple_display_count_v27; +static u32 external_suppression_count_v27; + +static bool trampoline_update_external_counts_v27(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + const u8 *request = in; + bool plug = request[0] & 1; + u32 apple_display = request[1] & 1; + u32 pipes = get_unaligned_le32(request + 2); + unsigned long flags; + + trace_iomfb_callback(dcp, tag, __func__); + + if (dcp->connector_type == DRM_MODE_CONNECTOR_eDP) + return true; + spin_lock_irqsave(&external_counts_lock_v27, flags); + if ((plug && (pipes > U32_MAX - external_pipe_count_v27 || + apple_display > U32_MAX - external_apple_display_count_v27)) || + (!plug && (pipes > dcp->external_pipes_v27 || + apple_display > dcp->external_displays_v27))) { + spin_unlock_irqrestore(&external_counts_lock_v27, flags); + dev_warn_ratelimited(dcp->dev, "invalid external display count update\n"); + return true; + } + dcp->external_pipes_v27 += plug ? pipes : -pipes; + dcp->external_displays_v27 += plug ? apple_display : -apple_display; + external_pipe_count_v27 += plug ? pipes : -pipes; + external_apple_display_count_v27 += plug ? apple_display : -apple_display; + spin_unlock_irqrestore(&external_counts_lock_v27, flags); + return true; +} + +static bool trampoline_get_external_counts_v27(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + const u8 *request = in; + unsigned long flags; + u32 counts[3]; + + trace_iomfb_callback(dcp, tag, __func__); + spin_lock_irqsave(&external_counts_lock_v27, flags); + counts[0] = external_pipe_count_v27; + counts[1] = external_apple_display_count_v27; + counts[2] = external_suppression_count_v27; + spin_unlock_irqrestore(&external_counts_lock_v27, flags); + + for (unsigned int i = 0; i < ARRAY_SIZE(counts); i++) + if (!(request[12 + i] & 1)) + put_unaligned_le32(counts[i], (u8 *)out + 4 * i); + return true; +} + +static bool trampoline_apply_suppression_v27(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + const u8 *request = in; + unsigned long flags; + + trace_iomfb_callback(dcp, tag, __func__); + spin_lock_irqsave(&external_counts_lock_v27, flags); + if (((request[0] & 1) && external_suppression_count_v27 == U32_MAX) || + (!(request[0] & 1) && !dcp->external_suppression_v27)) { + spin_unlock_irqrestore(&external_counts_lock_v27, flags); + return true; + } + dcp->external_suppression_v27 += (request[0] & 1) ? 1U : -1U; + external_suppression_count_v27 += (request[0] & 1) ? 1U : -1U; + spin_unlock_irqrestore(&external_counts_lock_v27, flags); + return true; +} + +static void external_counts_cleanup_v27(struct apple_dcp *dcp) +{ + unsigned long flags; + + spin_lock_irqsave(&external_counts_lock_v27, flags); + external_pipe_count_v27 -= dcp->external_pipes_v27; + external_apple_display_count_v27 -= dcp->external_displays_v27; + external_suppression_count_v27 -= dcp->external_suppression_v27; + dcp->external_pipes_v27 = 0; + dcp->external_displays_v27 = 0; + dcp->external_suppression_v27 = 0; + spin_unlock_irqrestore(&external_counts_lock_v27, flags); +} diff --git a/drivers/gpu/drm/apple/iomfb-peer-runtime-v27.inc b/drivers/gpu/drm/apple/iomfb-peer-runtime-v27.inc new file mode 100644 index 0000000000000..5d82c88266756 --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb-peer-runtime-v27.inc @@ -0,0 +1,126 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#include "iomfb-peer-v27.h" + +static void dcp_peer_swap_error(struct apple_dcp *owner, const char *where, u32 status) +{ + lockdep_assert_held(&owner->iomfb.lock); + dev_err(owner->dev, "paired swap %s failed: %#x\n", where, status); + dcp_peer_swap_failed_v27(owner); + /* One engine may already have accepted this frame. A failed peer ACK + * is not a DMA-retirement witness for either scanout. The clean queue + * retains its active transaction and next-scanout references on crash. */ + WRITE_ONCE(owner->crashed, true); + dcp_swap_watchdog_complete(owner); + dcp_present_failed(owner); + schedule_work(&owner->vblank_wq); +} + +static void dcp_peer_submit_error_on_owner(struct apple_dcp *owner, void *data, void *cookie) +{ + u32 *status = data; + dcp_peer_swap_error(owner, "peer submit", *status); +} + +static void dcp_peer_submitted(struct apple_dcp *dcp, void *data, void *cookie) +{ + struct apple_dcp *owner = cookie; + struct dcp_swap_submit_resp_v27 *resp = data; + u32 status = resp ? resp->ret : ~0U; + u32 id; + + if (owner != dcp) { + if (status && iomfb_queue_callback(owner, dcp_peer_submit_error_on_owner, + &status, sizeof(status), NULL)) { + WRITE_ONCE(owner->crashed, true); + schedule_work(&owner->vblank_wq); + } + return; + } + lockdep_assert_held(&owner->iomfb.lock); + if (READ_ONCE(owner->peer_frames.failed)) + return; + id = owner->swap.h17p.swap.swap_id; + if (status) { + dcp_peer_swap_error(owner, "primary submit", status); + return; + } + if (dcp_present_submit(owner, id, true)) { + dcp_peer_swap_error(owner, "submit state", -EPROTO); + return; + } + /* Per-endpoint receive ordering handles the primary ACK before its + * completion. The paired watermark still requires both completions. */ + owner->swap_start = ktime_get(); + owner->swap_submit_timestamp = dcp_swap_time(); + dcp_swap_watchdog_arm(owner); +} + +static void dcp_peer_started_on_owner(struct apple_dcp *owner, void *data, void *cookie) +{ + struct dcp_swap_start_resp_h17p *resp = data; + struct apple_dcp *peer = dcp_peer_scanout_v27(owner); + u32 id = owner->swap.h17p.swap.swap_id; + + lockdep_assert_held(&owner->iomfb.lock); + if (IS_ERR_OR_NULL(peer) || peer != cookie || resp->ret || resp->swap_id != id) { + dcp_peer_swap_error(owner, "start/ID", resp->ret ?: -EPROTO); + return; + } + /* Keep the logical atomic input intact. Both split wire records are + * primary-owned; dcp_push copies the peer's record into its own queue. */ + iomfb_serialize_present_h17p(&owner->present_h17p, &owner->swap.h17p); + if (!iomfb_apply_backlight_h17p(owner, &owner->swap.h17p, &owner->present_h17p)) { + dcp_peer_swap_error(owner, "surface preparation", -EINVAL); + return; + } + iomfb_serialize_present_v27(&owner->present_v27, &owner->swap.h17p); + owner->present_v27.swap.brightness_update = owner->present_h17p.swap[0x354] & 1; + owner->present_v27.swap.brightness_nits = get_unaligned_le64(owner->present_h17p.swap + 0x35e); + dcp_peer_split_swap_v27(&owner->present_v27, &owner->peer_present_v27, + owner->peer_tiling.width); + if (dcp_present_retires(owner)) + dcp_arm_retained_framebuffers(owner, id); + dcp_peer_swap_armed_v27(owner, id); + trace_iomfb_swap_submit(owner, id); + trace_iomfb_swap_submit(peer, id); + dcp_push(owner, false, &dcp_methods[dcpep_swap_submit], sizeof(owner->present_v27), + sizeof(struct dcp_swap_submit_resp_v27), &owner->present_v27, + dcp_peer_submitted, owner); + if (READ_ONCE(owner->crashed)) + return; + dcp_push(peer, false, &dcp_methods[dcpep_swap_submit], sizeof(owner->peer_present_v27), + sizeof(struct dcp_swap_submit_resp_v27), &owner->peer_present_v27, + dcp_peer_submitted, owner); + if (READ_ONCE(peer->crashed)) + dcp_peer_swap_error(owner, "peer publication", -EIO); +} + +static void dcp_peer_started(struct apple_dcp *peer, void *data, void *cookie) +{ + struct apple_dcp *owner = cookie; + struct dcp_swap_start_resp_h17p failed = { .ret = ~0U }; + + if (iomfb_queue_callback(owner, dcp_peer_started_on_owner, + data ?: &failed, sizeof(failed), peer)) { + WRITE_ONCE(owner->crashed, true); + schedule_work(&owner->vblank_wq); + } +} + +static bool dcp_peer_start_second(struct apple_dcp *owner, + struct dcp_swap_start_resp_h17p *resp) +{ + struct dcp_swap_start_req_h17p req = {}; + struct apple_dcp *peer = dcp_peer_scanout_v27(owner); + + if (!peer) + return false; + if (IS_ERR(peer) || resp->ret) { + dcp_peer_swap_error(owner, "primary start", resp->ret ?: -EIO); + return true; + } + owner->swap.h17p.swap.swap_id = resp->swap_id; + WRITE_ONCE(owner->started_swap_id, resp->swap_id); + dcp_swap_start(peer, false, &req, dcp_peer_started, owner); + return true; +} diff --git a/drivers/gpu/drm/apple/iomfb-peer-v27.h b/drivers/gpu/drm/apple/iomfb-peer-v27.h new file mode 100644 index 0000000000000..ea3b586e037c9 --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb-peer-v27.h @@ -0,0 +1,129 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef APPLE_IOMFB_PEER_V27_H +#define APPLE_IOMFB_PEER_V27_H + +#include +#include +#include +#include +#include "iomfb_plane.h" +#include "iomfb_v27.h" + +/* A364 and A365 share the video-data prefix, but not the final + * flags: A364 has a requires-merge boolean before three pointer-null flags; + * A365 has only two pointer-null flags and padding. + */ +struct dcp_peer_video_data_v27 { + u8 timing[0x50]; + u8 color[0x20]; +} __packed; + +struct dcp_peer_get_video_req_v27 { + struct dcp_peer_video_data_v27 video; + __le32 timing_id, color_id; + u8 requires_merge, timing_null, color_null, requires_merge_null; +} __packed; + +struct dcp_peer_get_video_resp_v27 { + struct dcp_peer_video_data_v27 video; + u8 requires_merge, success, padding[2]; +} __packed; + +struct dcp_peer_insert_video_req_v27 { + struct dcp_peer_video_data_v27 video; + __le32 timing_id, color_id; + u8 timing_null, color_null, padding[2]; +} __packed; + +struct dcp_peer_insert_video_resp_v27 { + struct dcp_peer_video_data_v27 video; + u8 success, padding[3]; +} __packed; + +static_assert(sizeof(struct dcp_peer_get_video_req_v27) == 0x7c); +static_assert(sizeof(struct dcp_peer_get_video_resp_v27) == 0x74); +static_assert(sizeof(struct dcp_peer_insert_video_req_v27) == 0x7c); +static_assert(sizeof(struct dcp_peer_insert_video_resp_v27) == 0x74); + +/* The digital-out mode setter. + * Keep vertical timings, color data and all unmodified fields intact. + * The signed horizontal offset uses division toward zero. + */ +static inline void dcp_peer_split_timing_v27(struct dcp_peer_video_data_v27 *v) +{ + static const u8 half_fields[] = { 4, 8, 12, 16, 20, 28, 68 }; + + for (unsigned int i = 0; i < sizeof(half_fields); i++) { + u8 *field = v->timing + half_fields[i]; + + put_unaligned_le32(get_unaligned_le32(field) >> 1, field); + } + put_unaligned_le32((s32)get_unaligned_le32(v->timing + 24) / 2, + v->timing + 24); +} + +/* VFTG splits a surface at the half-panel boundary. + * Source and destination use pixels here, as in the existing IOMFB plane + * path. Use rational arithmetic in the kernel rather than floating point; + * round a shared source boundary once so neither half loses/duplicates it. + * DRM has already clipped negative/offscreen destination coordinates. + */ +static inline bool dcp_peer_clip_surface_v27(const struct dcp_rect *src, + const struct dcp_rect *dst, u32 tile_x, u32 tile_width, + struct dcp_rect *tile_src, struct dcp_rect *tile_dst) +{ + u64 left = max_t(u64, dst->x, tile_x); + u64 right = min_t(u64, (u64)dst->x + dst->w, + (u64)tile_x + tile_width); + u64 first, last; + + memset(tile_src, 0, sizeof(*tile_src)); + memset(tile_dst, 0, sizeof(*tile_dst)); + if (!src->w || !src->h || !dst->w || !dst->h || left >= right) + return false; + first = div_u64((left - dst->x) * src->w + dst->w / 2, dst->w); + last = div_u64((right - dst->x) * src->w + dst->w / 2, dst->w); + if (first >= last || (u64)src->x + last > U32_MAX) + return false; + *tile_src = *src; + tile_src->x += first; + tile_src->w = last - first; + *tile_dst = *dst; + tile_dst->x = left - tile_x; + tile_dst->w = right - left; + return true; +} + + +/* Clone descriptors only: both DCPs use the same display DMA domain/IOVA. */ +static inline void dcp_peer_split_swap_v27(struct dcp_swap_submit_req_v27 *left, + struct dcp_swap_submit_req_v27 *right, + u32 tile_width) +{ + *right = *left; + for (unsigned int i = 0; i < ARRAY_SIZE(left->surf_null); i++) { + struct dcp_rect src = left->swap.src_rect[i]; + struct dcp_rect dst = left->swap.dst_rect[i]; + bool present = !left->surf_null[i]; + struct dcp_swap_submit_req_v27 *halves[] = {left, right}; + + for (unsigned int side = 0; side < 2; side++) { + struct dcp_swap_submit_req_v27 *req = halves[side]; + struct dcp_rect clipped_src, clipped_dst; + + if (present && dcp_peer_clip_surface_v27(&src, &dst, + side * tile_width, tile_width, &clipped_src, &clipped_dst)) { + req->swap.src_rect[i] = clipped_src; + req->swap.dst_rect[i] = clipped_dst; + } else { + req->surf_null[i] = true; + req->surf_iova[i] = 0; + req->swap.surf_ids[i] = 0; + memset(&req->swap.src_rect[i], 0, sizeof(req->swap.src_rect[i])); + memset(&req->swap.dst_rect[i], 0, sizeof(req->swap.dst_rect[i])); + } + } + } +} + +#endif diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index e749e958d3ce5..c15f77c0aaa7c 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -237,6 +237,10 @@ static int dcp_tx_offset(enum dcp_context_id id) case DCP_CONTEXT_OOBCB: case DCP_CONTEXT_OOBCMD: return 0x08000; + case DCP_CONTEXT_ASYNC_CB: + return 0x20000; + case DCP_CONTEXT_OOBASYNC_CB: + return 0x28000; default: return -EINVAL; } @@ -245,8 +249,10 @@ static int dcp_tx_offset(enum dcp_context_id id) static int dcp_channel_offset(enum dcp_context_id id) { switch (id) { + case DCP_CONTEXT_ASYNC_CB: case DCP_CONTEXT_ASYNC: return 0x40000; + case DCP_CONTEXT_OOBASYNC_CB: case DCP_CONTEXT_OOBASYNC: return 0x48000; case DCP_CONTEXT_CB: @@ -316,8 +322,10 @@ static struct dcp_channel *dcp_get_channel(struct apple_dcp *dcp, return &dcp->ch_oobcb; case DCP_CONTEXT_OOBCMD: return &dcp->ch_oobcmd; + case DCP_CONTEXT_ASYNC_CB: case DCP_CONTEXT_ASYNC: return &dcp->ch_async; + case DCP_CONTEXT_OOBASYNC_CB: case DCP_CONTEXT_OOBASYNC: return &dcp->ch_oobasync; default: @@ -353,7 +361,7 @@ static u8 dcp_pop_depth(u8 *depth) /* Older firmware and the H17G method profile retain their existing transport. */ static bool iomfb_uses_queue(struct apple_dcp *dcp) { - return dcp->fw_compat == DCP_FIRMWARE_H17P && + return (dcp->fw_compat == DCP_FIRMWARE_H17P || dcp->fw_compat == DCP_FIRMWARE_V_27_0) && dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G; } @@ -388,7 +396,8 @@ static int iomfb_enqueue_opaque_x(struct apple_dcp *dcp) if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL) return 0; - if (atomic_read(&dcp->iomfb.opaque_x_state) != IOMFB_OPAQUE_X_NEEDED) + if (dcp->fw_compat == DCP_FIRMWARE_V_27_0 || + atomic_read(&dcp->iomfb.opaque_x_state) != IOMFB_OPAQUE_X_NEEDED) return 0; if (dcp->iomfb.queued >= 32) return -EBUSY; @@ -432,7 +441,8 @@ static void iomfb_queue_advance(struct apple_dcp *dcp) /* The active operation may still be visible to firmware. */ return; } - if (!iomfb_channels_idle(dcp) || dcp->present_state_h17p.pending) + if (!iomfb_channels_idle(dcp) || dcp->present_state_h17p.pending || + READ_ONCE(dcp->iomfb.callback_count)) return; if (transaction) { @@ -478,7 +488,8 @@ static void iomfb_queue_work(struct work_struct *work) goto unlock; } if (dcp->iomfb.active || !iomfb_channels_idle(dcp) || - dcp->present_state_h17p.pending || list_empty(&dcp->iomfb.pending)) + dcp->present_state_h17p.pending || READ_ONCE(dcp->iomfb.callback_count) || + list_empty(&dcp->iomfb.pending)) goto unlock; transaction = list_first_entry(&dcp->iomfb.pending, @@ -571,12 +582,92 @@ static void iomfb_backlight_retry(struct work_struct *work) schedule_work(&dcp->bl_update_wq); } +/* A peer endpoint may finish a dependency while this endpoint has an active + * transaction. Continue on its serialized owner, not by taking another + * endpoint's IOMFB mutex from a receive callback. */ +struct iomfb_callback { + struct list_head link; + dcp_callback_t callback; + void *cookie; + u8 data[]; +}; + +int iomfb_queue_callback(struct apple_dcp *dcp, dcp_callback_t callback, + const void *data, size_t size, void *cookie) +{ + struct iomfb_callback *item; + unsigned long flags; + int ret = 0; + + /* This queue carries control replies and completion watermarks only. */ + if (!callback || size > 64 || (size && !data)) + return -EINVAL; + item = kzalloc(struct_size(item, data, size), GFP_KERNEL); + if (!item) + return -ENOMEM; + item->callback = callback; + item->cookie = cookie; + if (size) + memcpy(item->data, data, size); + spin_lock_irqsave(&dcp->iomfb.callback_lock, flags); + if (READ_ONCE(dcp->iomfb.stopped) || READ_ONCE(dcp->crashed)) { + ret = -ESHUTDOWN; + } else if (dcp->iomfb.callback_count >= 64) { + ret = -EBUSY; + } else { + list_add_tail(&item->link, &dcp->iomfb.callbacks); + dcp->iomfb.callback_count++; + /* Stop takes this lock before cancelling work, closing enqueue/stop. */ + schedule_work(&dcp->iomfb.callback_work); + } + spin_unlock_irqrestore(&dcp->iomfb.callback_lock, flags); + if (ret) + kfree(item); + return ret; +} + +static struct iomfb_callback *iomfb_pop_callback(struct apple_dcp *dcp) +{ + struct iomfb_callback *item = NULL; + unsigned long flags; + + spin_lock_irqsave(&dcp->iomfb.callback_lock, flags); + if (!list_empty(&dcp->iomfb.callbacks)) { + item = list_first_entry(&dcp->iomfb.callbacks, struct iomfb_callback, link); + list_del(&item->link); + dcp->iomfb.callback_count--; + } + spin_unlock_irqrestore(&dcp->iomfb.callback_lock, flags); + return item; +} + +static void iomfb_callback_work(struct work_struct *work) +{ + struct apple_dcp *dcp = container_of(work, struct apple_dcp, iomfb.callback_work); + struct iomfb_callback *item; + + mutex_lock(&dcp->iomfb.lock); + while ((item = iomfb_pop_callback(dcp))) { + if (!dcp->iomfb.stopped && !READ_ONCE(dcp->crashed)) { + WRITE_ONCE(dcp->iomfb.owner, current); + item->callback(dcp, item->data, item->cookie); + WRITE_ONCE(dcp->iomfb.owner, NULL); + } + kfree(item); + } + iomfb_queue_advance(dcp); + mutex_unlock(&dcp->iomfb.lock); +} + void iomfb_queue_init(struct apple_dcp *dcp) { mutex_init(&dcp->iomfb.lock); INIT_LIST_HEAD(&dcp->iomfb.pending); INIT_LIST_HEAD(&dcp->iomfb.dropped); INIT_WORK(&dcp->iomfb.work, iomfb_queue_work); + spin_lock_init(&dcp->iomfb.callback_lock); + INIT_LIST_HEAD(&dcp->iomfb.callbacks); + INIT_WORK(&dcp->iomfb.callback_work, iomfb_callback_work); INIT_DELAYED_WORK(&dcp->iomfb.timeout, iomfb_queue_timeout); INIT_DELAYED_WORK(&dcp->iomfb.backlight_retry, iomfb_backlight_retry); atomic_set(&dcp->iomfb.opaque_x_state, IOMFB_OPAQUE_X_WAITING); @@ -584,14 +675,22 @@ void iomfb_queue_init(struct apple_dcp *dcp) void iomfb_queue_stop(struct apple_dcp *dcp) { + struct iomfb_callback *item; + unsigned long flags; + if (!iomfb_uses_queue(dcp)) return; mutex_lock(&dcp->iomfb.lock); - dcp->iomfb.stopped = true; + spin_lock_irqsave(&dcp->iomfb.callback_lock, flags); + WRITE_ONCE(dcp->iomfb.stopped, true); + spin_unlock_irqrestore(&dcp->iomfb.callback_lock, flags); iomfb_discard_pending(dcp); mutex_unlock(&dcp->iomfb.lock); cancel_work_sync(&dcp->iomfb.work); + cancel_work_sync(&dcp->iomfb.callback_work); + while ((item = iomfb_pop_callback(dcp))) + kfree(item); cancel_delayed_work_sync(&dcp->iomfb.timeout); cancel_delayed_work_sync(&dcp->iomfb.backlight_retry); } @@ -612,7 +711,7 @@ bool iomfb_queue_drain(struct apple_dcp *dcp, unsigned long timeout) mutex_lock(&dcp->iomfb.lock); idle = READ_ONCE(dcp->crashed) || dcp->iomfb.stopped || (!dcp->iomfb.active && list_empty(&dcp->iomfb.pending) && - !dcp->present_state_h17p.pending); + !dcp->present_state_h17p.pending && !READ_ONCE(dcp->iomfb.callback_count)); mutex_unlock(&dcp->iomfb.lock); if (idle) return true; @@ -809,9 +908,9 @@ static int iomfb_queue_command(struct apple_dcp *dcp, bool oob, } /* Call a DCP function given by a tag */ -void dcp_push(struct apple_dcp *dcp, bool oob, const struct dcp_method_entry *call, - u32 in_len, u32 out_len, void *data, dcp_callback_t cb, - void *cookie) +static void dcp_push_context(struct apple_dcp *dcp, enum dcp_context_id context, + const struct dcp_method_entry *call, u32 in_len, + u32 out_len, void *data, dcp_callback_t cb, void *cookie) { struct dcp_method_entry resolved = *call; @@ -823,14 +922,6 @@ void dcp_push(struct apple_dcp *dcp, bool oob, const struct dcp_method_entry *ca } #endif - /* Reply chains stay on the serialized receiver; callers enqueue copies. */ - if (iomfb_uses_queue(dcp) && READ_ONCE(dcp->iomfb.owner) != current) { - if (iomfb_queue_command(dcp, oob, call, in_len, out_len, - data, cb, cookie)) - WRITE_ONCE(dcp->crashed, true); - return; - } - if (dcp_fail_stop(dcp) && READ_ONCE(dcp->crashed)) return; @@ -839,7 +930,6 @@ void dcp_push(struct apple_dcp *dcp, bool oob, const struct dcp_method_entry *ca memcpy(resolved.tag, call->tag_h17g, sizeof(resolved.tag)); call = &resolved; - enum dcp_context_id context = dcp_call_context(dcp, oob); struct dcp_channel *ch = dcp_get_channel(dcp, context); struct dcp_packet_header header = { @@ -893,6 +983,54 @@ void dcp_push(struct apple_dcp *dcp, bool oob, const struct dcp_method_entry *ca dcpep_msg(context, data_len, offset)); } +void dcp_push(struct apple_dcp *dcp, bool oob, + const struct dcp_method_entry *call, u32 in_len, u32 out_len, + void *data, dcp_callback_t cb, void *cookie) +{ + /* Reply chains stay on the serialized receiver; callers enqueue copies. */ + if (iomfb_uses_queue(dcp) && READ_ONCE(dcp->iomfb.owner) != current) { + if (iomfb_queue_command(dcp, oob, call, in_len, out_len, + data, cb, cookie)) + WRITE_ONCE(dcp->crashed, true); + return; + } + + dcp_push_context(dcp, dcp_call_context(dcp, oob), call, + in_len, out_len, data, cb, cookie); +} + +/* A nested query must return on the firmware-initiated stream it answers. */ +int dcp_push_nested(struct apple_dcp *dcp, enum dcp_context_id origin, + const struct dcp_method_entry *call, u32 in_len, u32 out_len, + void *data, dcp_callback_t cb, void *cookie) +{ + enum dcp_context_id context; + struct dcp_channel *ch; + + if (READ_ONCE(dcp->iomfb.owner) != current || READ_ONCE(dcp->crashed)) + return -ESHUTDOWN; + switch (origin) { + case DCP_CONTEXT_CB: + case DCP_CONTEXT_OOBCB: + context = origin; + break; + case DCP_CONTEXT_ASYNC: + context = DCP_CONTEXT_ASYNC_CB; + break; + case DCP_CONTEXT_OOBASYNC: + context = DCP_CONTEXT_OOBASYNC_CB; + break; + default: + return -EINVAL; + } + ch = dcp_get_channel(dcp, context); + if (!ch || !ch->depth || ch->depth >= DCP_MAX_CALL_DEPTH || + (u64)sizeof(struct dcp_packet_header) + in_len + out_len > 0x8000) + return -EINVAL; + dcp_push_context(dcp, context, call, in_len, out_len, data, cb, cookie); + return READ_ONCE(dcp->crashed) ? -EPROTO : 0; +} + /* Parse a callback tag "D123" into the ID 123. Returns -EINVAL on failure. */ int dcp_parse_tag(char tag[4]) { @@ -1530,6 +1668,7 @@ static void dcpep_handle_cb(struct apple_dcp *dcp, enum dcp_context_id context, bool handled, zero_reply; if ((dcp->fw_compat == DCP_FIRMWARE_H17P || + dcp->fw_compat == DCP_FIRMWARE_V_27_0 || dcp->fw_compat == DCP_FIRMWARE_V_26_6_DESKTOP) && (length < sizeof(*hdr) || !ch || ch->depth >= DCP_MAX_CALL_DEPTH || offset >= 0x8000 || length > 0x8000 - offset)) { @@ -1567,13 +1706,15 @@ static void dcpep_handle_cb(struct apple_dcp *dcp, enum dcp_context_id context, dcp->crashed = true; return; } - if (dcp->fw_compat == DCP_FIRMWARE_H17P && + if ((dcp->fw_compat == DCP_FIRMWARE_H17P || dcp->fw_compat == DCP_FIRMWARE_V_27_0) && (tag < 0 || tag >= IOMFB_MAX_CB || (u64)sizeof(*hdr) + hdr->in_len + hdr->out_len != length || (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G && handled && - !iomfb_check_callback_h17p(dcp, tag, hdr->in_len, hdr->out_len)))) { - dev_err(dev, "unsupported IOMFB callback %d (%u, %u)\n", + !(dcp->fw_compat == DCP_FIRMWARE_V_27_0 ? + iomfb_v27_callback_size_valid(tag, hdr->in_len, hdr->out_len) : + iomfb_check_callback_h17p(dcp, tag, hdr->in_len, hdr->out_len))))) { + dev_err(dev, "unqualified IOMFB callback %d (%u, %u)\n", tag, hdr->in_len, hdr->out_len); dcp->crashed = true; return; @@ -1609,6 +1750,7 @@ static void dcpep_handle_cb(struct apple_dcp *dcp, enum dcp_context_id context, READ_ONCE(dcp->desktop->modeset_active)) WRITE_ONCE(dcp->desktop->modeset_callback_error, true); if (dcp->fw_compat == DCP_FIRMWARE_H17P || + dcp->fw_compat == DCP_FIRMWARE_V_27_0 || dcp->fw_compat == DCP_FIRMWARE_V_26_6_DESKTOP) { if (hdr->out_len) memset(out, 0, hdr->out_len); @@ -1675,7 +1817,7 @@ static void dcpep_handle_ack(struct apple_dcp *dcp, enum dcp_context_id context) * its output cannot be trusted. The tag is left out of the check: it * has only been seen preserved on 13.5 firmware. */ - if (dcp->fw_compat != DCP_FIRMWARE_H17P) { + if ((dcp->fw_compat != DCP_FIRMWARE_H17P && dcp->fw_compat != DCP_FIRMWARE_V_27_0)) { record = out - sent->in_len - sizeof(*record); if (record->in_len != sent->in_len || record->out_len != sent->out_len) { @@ -2901,6 +3043,9 @@ int dcp_crtc_atomic_modeset(struct drm_crtc *crtc, case DCP_FIRMWARE_H17P: ret = iomfb_modeset_h17p(dcp, crtc_state); break; + case DCP_FIRMWARE_V_27_0: + ret = iomfb_modeset_v27(dcp, crtc_state); + break; default: WARN_ONCE(true, "Unexpected firmware version: %u\n", dcp->fw_compat); @@ -3185,8 +3330,9 @@ bool iomfb_present_complete_h17p(struct apple_dcp *dcp) if (transaction->brightness_only) return false; } - atomic_cmpxchg(&dcp->iomfb.opaque_x_state, IOMFB_OPAQUE_X_WAITING, - IOMFB_OPAQUE_X_NEEDED); + if (dcp->fw_compat != DCP_FIRMWARE_V_27_0) + atomic_cmpxchg(&dcp->iomfb.opaque_x_state, IOMFB_OPAQUE_X_WAITING, + IOMFB_OPAQUE_X_NEEDED); iomfb_scanout_complete_h17p(dcp); return true; } @@ -3266,7 +3412,10 @@ static void iomfb_backlight_start(struct apple_dcp *dcp, return; dcp->swap.h17p = scanout->request; - iomfb_present_backlight_h17p(dcp); + if (dcp->fw_compat == DCP_FIRMWARE_V_27_0) + iomfb_present_backlight_v27(dcp); + else + iomfb_present_backlight_h17p(dcp); } static void iomfb_backlight_release(struct iomfb_transaction *transaction) @@ -3324,7 +3473,10 @@ static void iomfb_atomic_start(struct apple_dcp *dcp, return; } dcp->iomfb.next_scanout = atomic->scanout; - iomfb_flush_h17p(dcp, atomic->crtc, atomic->state); + if (dcp->fw_compat == DCP_FIRMWARE_V_27_0) + iomfb_flush_v27(dcp, atomic->crtc, atomic->state); + else + iomfb_flush_h17p(dcp, atomic->crtc, atomic->state); } static void iomfb_atomic_release(struct iomfb_transaction *transaction) @@ -3484,6 +3636,9 @@ void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_commit *commit) case DCP_FIRMWARE_H17P: iomfb_flush_h17p(dcp, crtc, commit); break; + case DCP_FIRMWARE_V_27_0: + iomfb_flush_v27(dcp, crtc, commit); + break; default: WARN_ONCE(true, "Unexpected firmware version: %u\n", dcp->fw_compat); break; @@ -3508,6 +3663,9 @@ static void iomfb_start(struct apple_dcp *dcp) case DCP_FIRMWARE_H17P: iomfb_start_h17p(dcp); break; + case DCP_FIRMWARE_V_27_0: + iomfb_start_v27(dcp); + break; default: WARN_ONCE(true, "Unexpected firmware version: %u\n", dcp->fw_compat); break; @@ -3583,7 +3741,7 @@ int iomfb_start_rtkit(struct apple_dcp *dcp) * has nothing answering it on the AP side, and without this one the * display DART faults. */ - if (dcp->fw_compat == DCP_FIRMWARE_H17P && + if ((dcp->fw_compat == DCP_FIRMWARE_H17P || dcp->fw_compat == DCP_FIRMWARE_V_27_0) && dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17P_EXTERNAL) { ret = apple_rtkit_start_ep(dcp->rtk, REMOTE_ALLOC_ENDPOINT); if (ret) @@ -3628,6 +3786,9 @@ void iomfb_shutdown(struct apple_dcp *dcp) case DCP_FIRMWARE_H17P: iomfb_shutdown_h17p(dcp); break; + case DCP_FIRMWARE_V_27_0: + iomfb_shutdown_v27(dcp); + break; default: WARN_ONCE(true, "Unexpected firmware version: %u\n", dcp->fw_compat); break; diff --git a/drivers/gpu/drm/apple/iomfb.h b/drivers/gpu/drm/apple/iomfb.h index 2ae15b13b8d43..1021825209fb3 100644 --- a/drivers/gpu/drm/apple/iomfb.h +++ b/drivers/gpu/drm/apple/iomfb.h @@ -4,6 +4,8 @@ #ifndef __APPLE_DCPEP_H__ #define __APPLE_DCPEP_H__ +#include +#include #include #include "version_utils.h" @@ -15,6 +17,7 @@ enum dcp_context_id { /* Callback */ DCP_CONTEXT_CB = 0, + DCP_CONTEXT_ASYNC_CB = 1, /* Command */ DCP_CONTEXT_CMD = 2, @@ -24,6 +27,7 @@ enum dcp_context_id { /* Out-of-band callback */ DCP_CONTEXT_OOBCB = 4, + DCP_CONTEXT_OOBASYNC_CB = 5, /* Out-of-band command */ DCP_CONTEXT_OOBCMD = 6, @@ -184,6 +188,8 @@ enum dcpep_method { * H17P pipe-configuration calls made before the first swap. Most * names are unknown; they are identified by method number. */ + dcpep_pipe_query_353, + dcpep_pipe_set_352, dcpep_pipe_cfg_415, dcpep_pipe_cfg_031, dcpep_pipe_cfg_414, @@ -476,6 +482,44 @@ struct dcp_hotplug_resp_v26_6 { u8 opaque[0x4c]; } __packed; +/* + * A name and a serialized dictionary: the firmware's analytics event + * (D113 on 27.x, D114 on desktop 26.6). + */ +struct dcp_dict_passthrough_req { + u8 key[0x40]; + u8 value[0x1000]; + u8 value_null; + u8 padding[3]; +} __packed; + +struct dcp_dict_passthrough_resp { + u8 value[0x1000]; + u8 ret; + u8 padding[3]; +} __packed; + +static_assert(sizeof(struct dcp_dict_passthrough_req) == 0x1044); +static_assert(sizeof(struct dcp_dict_passthrough_resp) == 0x1004); + +/* + * The analytics event passes its dictionary by pointer, so the firmware + * deserializes the reply's copy when the call returns, and stops if that + * fails. Return the dictionary as it came, and report the event not sent. + * A zeroed reply is not a dictionary. + */ +static inline void +dcp_dict_passthrough_reply(const struct dcp_dict_passthrough_req *req, + struct dcp_dict_passthrough_resp *resp) +{ + if (req->value_null) + memset(resp->value, 0, sizeof(resp->value)); + else + memcpy(resp->value, req->value, sizeof(resp->value)); + resp->ret = false; + memset(resp->padding, 0, sizeof(resp->padding)); +} + /* A boolean argument, padded to four bytes. */ struct dcp_bool_arg { u8 value; diff --git a/drivers/gpu/drm/apple/iomfb_internal.h b/drivers/gpu/drm/apple/iomfb_internal.h index b34a1171123e3..5d32a1c034e91 100644 --- a/drivers/gpu/drm/apple/iomfb_internal.h +++ b/drivers/gpu/drm/apple/iomfb_internal.h @@ -118,3 +118,6 @@ u8 dcp_crtc_eotf(struct apple_dcp *dcp, struct drm_crtc *crtc, int dcp_crtc_check_hdr(struct apple_dcp *dcp, struct drm_crtc *crtc, struct drm_atomic_commit *commit); void dcp_hdr_metadata_forget(struct apple_dcp *dcp); +int dcp_push_nested(struct apple_dcp *dcp, enum dcp_context_id origin, + const struct dcp_method_entry *call, u32 in_len, u32 out_len, + void *data, dcp_callback_t cb, void *cookie); diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 80d341e7eb2e1..b400dc07cf168 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -32,6 +33,7 @@ #include "dcp-edr.h" #include "iomfb.h" #include "iomfb_internal.h" +#include "iomfb-dfb.h" #include "parser.h" #include "dcp-refresh.h" #include "dcp-property.h" @@ -369,10 +371,24 @@ static void dcp_swap_submit(struct apple_dcp *dcp, bool oob, cb(dcp, NULL, cookie); return; } +#if (DCP_FW_H17(DCP_FW_VER) && DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0)) + iomfb_serialize_present_v27(&dcp->present_v27, request); + dcp->present_v27.swap.brightness_update = + dcp->present_h17p.swap[0x354] & 1; + dcp->present_v27.swap.brightness_nits = + get_unaligned_le64(dcp->present_h17p.swap + 0x35e); + data = &dcp->present_v27; +#else data = &dcp->present_h17p; +#endif } - dcp_push(dcp, oob, &dcp_methods[dcpep_swap_submit], sizeof(*request), + dcp_push(dcp, oob, &dcp_methods[dcpep_swap_submit], +#if (DCP_FW_H17(DCP_FW_VER) && DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0)) + sizeof(struct dcp_swap_submit_req_v27), +#else + sizeof(*request), +#endif sizeof(struct dcp_swap_submit_resp_h17p), (void *)data, cb, cookie); } #else @@ -407,7 +423,7 @@ DCP_THUNK_INOUT(dcp_set_power_state, dcpep_set_power_state, * programmed, which is the panel's native mode, instead of requesting one. */ #if DCP_FW_H17(DCP_FW_VER) -#define DCP_INHERIT_BOOT_MODE true +#define DCP_INHERIT_BOOT_MODE (DCP_FW_VER < DCP_FW_VERSION(27, 0, 0)) #else #define DCP_INHERIT_BOOT_MODE false #endif @@ -420,7 +436,7 @@ DCP_THUNK_INOUT(dcp_set_power_state, dcpep_set_power_state, * still sends its clear swap first. */ #if DCP_FW_H17(DCP_FW_VER) -#define DCP_HAS_CLIENT_TEARDOWN false +#define DCP_HAS_CLIENT_TEARDOWN ((DCP_FW_H17(DCP_FW_VER) && DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0))) #else #define DCP_HAS_CLIENT_TEARDOWN true #endif @@ -600,29 +616,27 @@ static u32 dcpep_cb_zero(struct apple_dcp *dcp) return 0; } -static void dcpep_cb_swap_complete(struct apple_dcp *dcp, - struct DCP_FW_NAME(dc_swap_complete_resp) *resp) +static void dcp_report_swap_complete(struct apple_dcp *dcp, u32 id, ktime_t now) { - ktime_t now = ktime_get(); bool retire = true; #if DCP_FW_DESKTOP_26(DCP_FW_VER) - if (dcp_peer_swap_complete_26_6(dcp, resp->swap_id)) + if (dcp_peer_swap_complete_26_6(dcp, id)) return; #endif #if DCP_FW_H17(DCP_FW_VER) if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G && - !dcp_present_complete_h17p(&dcp->present_state_h17p, resp->swap_id)) { + !dcp_present_complete_h17p(&dcp->present_state_h17p, id)) { /* * Abort semantics are not captured on H17P. If an aborted * present completes after all, its framebuffers were already * unarmed and its event signalled; record it and carry on. */ if (dcp_present_was_aborted_h17p(&dcp->present_state_h17p, - resp->swap_id)) { + id)) { dev_warn_ratelimited(dcp->dev, "completion for aborted present %u ignored\n", - resp->swap_id); + id); return; } /* @@ -631,13 +645,13 @@ static void dcpep_cb_swap_complete(struct apple_dcp *dcp, * in flight now is a different one. */ if (dcp_present_was_retired_h17p(&dcp->present_state_h17p, - resp->swap_id)) { + id)) { dev_warn_ratelimited(dcp->dev, "completion for retired present %u ignored\n", - resp->swap_id); + id); return; } - dev_err(dcp->dev, "unexpected present completion %u\n", resp->swap_id); + dev_err(dcp->dev, "unexpected present completion %u\n", id); dcp->crashed = true; return; } @@ -646,36 +660,65 @@ static void dcpep_cb_swap_complete(struct apple_dcp *dcp, #endif #if DCP_FW_DESKTOP_26(DCP_FW_VER) /* Late or duplicate completions cannot retire the replacement's flip. */ - if (!dcp_desktop_present_complete(dcp, resp->swap_id)) + if (!dcp_desktop_present_complete(dcp, id)) return; - dcp_backlight_desktop_finish(dcp, resp->swap_id, false, true); + dcp_backlight_desktop_finish(dcp, id, false, true); #else /* Nor can a lost flip's on firmware that does not track its presents. */ if ((!DCP_FW_H17(DCP_FW_VER) || dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G) && - dcp_swap_lost(dcp, resp->swap_id)) + dcp_swap_lost(dcp, id)) return; #endif - trace_iomfb_swap_complete(dcp, resp->swap_id); - dcp->last_swap_id = resp->swap_id; + dcp->last_swap_id = id; dcp->have_swap_complete = true; dcp_swap_watchdog_complete(dcp); if (retire) { - dcp_release_retained_framebuffers(dcp, resp->swap_id); + dcp_release_retained_framebuffers(dcp, id); dcp_drm_crtc_page_flip(dcp, now); } if (READ_ONCE(dcp->crc_enabled)) { #if DCP_FW_H17(DCP_FW_VER) if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) { - iomfb_queue_crc_h17p(dcp, resp->swap_id); + iomfb_queue_crc_h17p(dcp, id); return; } #endif u32 crc32 = 0; - drm_crtc_add_crc_entry(&dcp->crtc->base, true, resp->swap_id, &crc32); + drm_crtc_add_crc_entry(&dcp->crtc->base, true, id, &crc32); } } +#if (DCP_FW_H17(DCP_FW_VER) && DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0)) +struct dcp_peer_completion { u32 id; ktime_t time; }; +static void dcp_peer_report_on_owner(struct apple_dcp *owner, void *data, void *cookie) +{ + struct dcp_peer_completion *completion = data; + dcp_report_swap_complete(owner, completion->id, completion->time); +} +static void dcp_peer_queue_completion(struct apple_dcp *owner, u32 id, ktime_t time) +{ + struct dcp_peer_completion completion = { id, time }; + if (iomfb_queue_callback(owner, dcp_peer_report_on_owner, &completion, + sizeof(completion), NULL)) { + WRITE_ONCE(owner->crashed, true); + schedule_work(&owner->vblank_wq); + } +} +#endif + +static void dcpep_cb_swap_complete(struct apple_dcp *dcp, + struct DCP_FW_NAME(dc_swap_complete_resp) *resp) +{ + ktime_t now = ktime_get(); + trace_iomfb_swap_complete(dcp, resp->swap_id); +#if (DCP_FW_H17(DCP_FW_VER) && DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0)) + if (dcp_peer_swap_complete_v27(dcp, resp->swap_id, now, dcp_peer_queue_completion)) + return; +#endif + dcp_report_swap_complete(dcp, resp->swap_id, now); +} + /* special */ static void complete_vi_set_temperature_hint(struct apple_dcp *dcp, void *out, void *cookie) { @@ -716,7 +759,7 @@ static_assert(sizeof(struct dcp_h17p_dfb_surface_req) == 0x230); DCP_THUNK_INOUT(dcp_register_dfb_surface, dcpep_register_dfb_surface, struct dcp_h17p_dfb_surface_req, u32); -static void dcp_ack_dfb_surface(struct apple_dcp *dcp) +static void __maybe_unused dcp_ack_dfb_surface(struct apple_dcp *dcp) { struct dcp_channel *ch = &dcp->ch_cb; u8 *succ = ch->output[ch->depth - 1]; @@ -731,7 +774,7 @@ static void dcp_ack_dfb_surface(struct apple_dcp *dcp) * default surface ID. The bytes at +0x29..+0x2c are an opaque word whose * meaning is not known. */ -static void dcp_fill_dfb_surface(struct apple_dcp *dcp, u8 *s) +static void __maybe_unused dcp_fill_dfb_surface(struct apple_dcp *dcp, u8 *s) { s[0x002] = 0x01; memcpy(s + 0x00b, "8a3b", 4); @@ -759,7 +802,14 @@ static void dcp_fill_dfb_surface(struct apple_dcp *dcp, u8 *s) */ static void complete_dfb_surface(struct apple_dcp *dcp, void *out, void *cookie) { +#if (DCP_FW_H17(DCP_FW_VER) && DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0)) + struct dcp_channel *ch = &dcp->ch_cb; + + *(u8 *)ch->output[ch->depth - 1] = out && !get_unaligned_le32(out); + dcp_ack(dcp, DCP_CONTEXT_CB); +#else dcp_ack_dfb_surface(dcp); +#endif } static bool iomfbep_cb_create_dfb_surface(struct apple_dcp *dcp, int tag, @@ -779,12 +829,48 @@ static bool iomfbep_cb_create_dfb_surface(struct apple_dcp *dcp, int tag, if (!req) return true; + +#if (DCP_FW_H17(DCP_FW_VER) && DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0)) + { + struct dcp_dfb_dimensions_h17p dim; + struct dcp_surface *surface = (void *)req->surface; + u32 format, allocation, bpe; + bool alpha_plane; + int ret = -EINVAL; + + memcpy(&dim, in, sizeof(dim)); + if (dcp->dfb_format_valid && dcp->dfb_compression_valid) + ret = apple_dcp_dfb_allocation(dim.width, dim.height, + dcp->dfb_format, dcp->dfb_compression, + dcp->dfb_region_size, &format, &allocation); + if (ret) { + dev_err(dcp->dev, "unsupported DFB layout %ux%u format=%#x compression=%u (%d)\n", + dim.width, dim.height, dcp->dfb_format, + dcp->dfb_compression, ret); + *(u8 *)out = false; + kfree(req); + return true; + } + + apple_dcp_dfb_format(dcp->dfb_format, dcp->dfb_compression, + &format, &bpe, &alpha_plane); + surface->is_premultiplied = 1; + surface->format = format; + surface->pix_size = dcp_surface_element_geometry(dcp) ? bpe : 1; + surface->pel_w = 1; + surface->pel_h = 1; + surface->buf_size = allocation; + surface->surface_id = 5; + surface->has_comp = 1; + surface->has_planes = 1; + } +#else dcp_fill_dfb_surface(dcp, req->surface); +#endif dcp_register_dfb_surface(dcp, false, req, complete_dfb_surface, NULL); kfree(req); - /* deferred ACK: the A472 completion reports success for us */ return false; } #endif @@ -843,6 +929,9 @@ static void iomfb_cb_pr_publish(struct apple_dcp *dcp, struct iomfb_property *pr switch (prop->id) { #if !DCP_FW_DESKTOP_26(DCP_FW_VER) /* The desktop 26.x firmware publishes no panel level here. */ + #if (DCP_FW_H17(DCP_FW_VER) && DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0)) + case 20: +#else case IOMFB_PROPERTY_NITS_V13_3: /* * 13.x firmware publishes another property under the 14.7 ID, with @@ -851,6 +940,7 @@ static void iomfb_cb_pr_publish(struct apple_dcp *dcp, struct iomfb_property *pr */ #if DCP_FW_VERSION(14, 7, 0) <= DCP_FW_VER case IOMFB_PROPERTY_NITS_V14_7: +#endif #endif { #if DCP_FW_H17(DCP_FW_VER) @@ -1285,7 +1375,8 @@ static struct DCP_FW_NAME(dcp_map_reg_resp) dcpep_cb_map_reg(struct apple_dcp *d * dma_map_resource() them instead fails and hands the firmware * DMA_MAPPING_ERROR as a device address. */ - dma_addr_t dva = rsrc->start; + dma_addr_t dva = dcp->fw_compat == DCP_FIRMWARE_V_27_0 ? + dcp->disp_firmware_base[req->index] : rsrc->start; #elif DCP_FW_DESKTOP_26(DCP_FW_VER) /* * The desktop 26.x firmware takes the register windows' @@ -1300,7 +1391,12 @@ static struct DCP_FW_NAME(dcp_map_reg_resp) dcpep_cb_map_reg(struct apple_dcp *d #endif return (struct DCP_FW_NAME(dcp_map_reg_resp)){ - .addr = rsrc->start, + .addr = +#if (DCP_FW_H17(DCP_FW_VER) && DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0)) + dcp->disp_firmware_base[req->index], +#else + rsrc->start, +#endif .length = resource_size(rsrc), #if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) .dva = dva, @@ -1497,6 +1593,17 @@ static bool dcpep_process_chunks(struct apple_dcp *dcp, dev_warn(dcp->dev, "TimingElements without valid modes!\n"); dcp_180hz_notify(dcp); dcp_firmware_metadata_timing(dcp); +#if (DCP_FW_H17(DCP_FW_VER) && DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0)) + if (dcp->peer_reserved) + for (unsigned int i = 0; i < dcp->nr_modes; i++) { + struct dcp_display_mode *m = &dcp->modes[i]; + + if ((u64)m->mode.clock * 1000 > dcp->dptxport[0].caps.total_rate) + dev_info(dcp->dev, "peer mode timing=%u color=%u " DRM_MODE_FMT "\n", + m->timing_mode_id, m->color_mode_id, + DRM_MODE_ARG(&m->mode)); + } +#endif } else if (!strcmp(req->key, "DisplayAttributes")) { struct dcp_parse_ctx metadata_ctx = ctx; @@ -1642,6 +1749,27 @@ static void boot_1_11(struct apple_dcp *dcp, void *out, void *cookie) static void boot_1_10(struct apple_dcp *dcp, void *out, void *cookie) { +#if (DCP_FW_H17(DCP_FW_VER) && DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0)) + struct dcp_dfb_dimensions_h17p dim; + + if (!out) + return; + memcpy(&dim, out, sizeof(dim)); + + if (dim.width > 16384 || dim.height > 16384 + ) { + struct dcp_channel *ch = &dcp->ch_cb; + + dev_err(dcp->dev, "invalid default framebuffer dimensions %ux%u\n", + dim.width, dim.height); + *(u8 *)ch->output[ch->depth - 1] = false; + dcp_ack(dcp, DCP_CONTEXT_CB); + return; + } + + dcp_set_dfb_dimensions(dcp, false, &dim, boot_1_11, NULL); + +#else struct dcp_dfb_dimensions_h17p dim; /* @@ -1662,12 +1790,25 @@ static void boot_1_10(struct apple_dcp *dcp, void *out, void *cookie) dim.width, dim.height); dcp_set_dfb_dimensions(dcp, false, &dim, boot_1_11, NULL); + +#endif } static void boot_1_9(struct apple_dcp *dcp, void *out, void *cookie) { u32 zero = 0; +#if (DCP_FW_H17(DCP_FW_VER) && DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0)) + struct dcp_dfb_dimensions_h17p dim; + + if (!out) + return; + memcpy(&dim, out, sizeof(dim)); + if (dim.width && dim.height) { + boot_2(dcp, out, cookie); + return; + } +#endif dcp_get_dfb_state(dcp, false, &zero, boot_1_10, NULL); } @@ -1675,6 +1816,12 @@ static void boot_1_8(struct apple_dcp *dcp, void *out, void *cookie) { struct dcp_dfb_layout_req_h17p req = {}; +#if (DCP_FW_H17(DCP_FW_VER) && DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0)) + if (!out) + return; + dcp->dfb_format_valid = *((u8 *)out + 0x28) & 1; + memcpy(&dcp->dfb_format, out, sizeof(dcp->dfb_format)); +#endif dcp_get_dfb_layout(dcp, false, &req, boot_1_9, NULL); } @@ -1682,6 +1829,12 @@ static void boot_1_7(struct apple_dcp *dcp, void *out, void *cookie) { struct dcp_dfb_info_h17p info = {}; +#if (DCP_FW_H17(DCP_FW_VER) && DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0)) + if (!out) + return; + dcp->dfb_compression_valid = *((u8 *)out + 4) & 1; + memcpy(&dcp->dfb_compression, out, sizeof(dcp->dfb_compression)); +#endif dcp_get_dfb_info(dcp, false, &info, boot_1_8, NULL); } @@ -1689,6 +1842,16 @@ static void boot_1_6(struct apple_dcp *dcp, void *out, void *cookie) { u64 zero = 0; +#if (DCP_FW_H17(DCP_FW_VER) && DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0)) + if (!out) + return; + if (!(*(u8 *)out & 1)) { + struct dcp_dfb_layout_req_h17p req = {}; + + dcp_get_dfb_layout(dcp, false, &req, boot_1_9, NULL); + return; + } +#endif dcp_get_dfb_compression_info(dcp, false, &zero, boot_1_7, NULL); } #endif @@ -1869,6 +2032,7 @@ static void release_swap_cookie(struct kref *ref) */ static void dcp_h17p_prepare_swap(struct apple_dcp *dcp) { +#if DCP_FW_VER < DCP_FW_VERSION(27, 0, 0) struct DCP_FW_NAME(dcp_swap_submit_req) *req = &DCP_FW_SWAP(dcp); unsigned int i; @@ -1918,6 +2082,7 @@ static void dcp_h17p_prepare_swap(struct apple_dcp *dcp) memmove(r + off - 2, r + off, n - off); memset(r + n - 2, 0, 2); } +#endif } #else static void dcp_h17p_prepare_swap(struct apple_dcp *dcp) { } @@ -2184,7 +2349,12 @@ static void dcp_on_set_power_state(struct apple_dcp *dcp, void *out, void *cooki static void dcp_on_set_parameter(struct apple_dcp *dcp, void *out, void *cookie) { struct dcp_set_parameter_dcp param = { - .param = IOMFBPARAM_ADAPTIVE_SYNC, + .param = +#if (DCP_FW_H17(DCP_FW_VER) && DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0)) + 15, +#else + IOMFBPARAM_ADAPTIVE_SYNC, +#endif .value = { 0 }, #if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) .count = 3, @@ -2379,6 +2549,16 @@ void DCP_FW_NAME(iomfb_poweroff)(struct apple_dcp *dcp) goto peer_stopped; #endif +#if (DCP_FW_H17(DCP_FW_VER) && DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0)) + ret = dcp_peer_stop_v27(dcp); + if (ret < 0) { + dev_err(dcp->dev, "paired power-off failed: %d\n", ret); + return; + } + if (ret > 0) + goto close_client; +#endif + cookie = kzalloc(sizeof(*cookie), GFP_KERNEL); if (!cookie) return; @@ -2420,6 +2600,9 @@ void DCP_FW_NAME(iomfb_poweroff)(struct apple_dcp *dcp) #if DCP_FW_DESKTOP_26(DCP_FW_VER) peer_stopped: +#endif +#if (DCP_FW_H17(DCP_FW_VER) && DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0)) +close_client: #endif poff_cookie = kzalloc(sizeof(*poff_cookie), GFP_KERNEL); if (!poff_cookie) @@ -2546,6 +2729,14 @@ void DCP_FW_NAME(iomfb_sleep)(struct apple_dcp *dcp) struct dcp_wait_cookie *cookie; +#if (DCP_FW_H17(DCP_FW_VER) && DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0)) + ret = dcp_peer_stop_v27(dcp); + if (ret < 0) { + dev_err(dcp->dev, "paired sleep failed: %d\n", ret); + return; + } +#endif + #if DCP_FW_H17(DCP_FW_VER) if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) return; @@ -2659,6 +2850,12 @@ static void dcpep_cb_hotplug(struct apple_dcp *dcp, u64 *connected) action |= DCP_HOTPLUG_NOTIFY; } dcp_handle_hotplug_actions(dcp, action); + /* Fresh modes and positive firmware HPD, not lane-count negotiation, + * make IOMFB ready for the resume modeset. */ + if (*connected && dcp->fw_compat == DCP_FIRMWARE_V_27_0 && + dcp_is_typec_output(dcp)) + complete_all(&dcp->typec_display_ready); + } #if DCP_FW_DESKTOP_26(DCP_FW_VER) @@ -2892,6 +3089,43 @@ TRAMPOLINE_INOUT(trampoline_set_tiling_state_v26_6, dcpep_cb_set_tiling_state_v2 #endif TRAMPOLINE_OUT(trampoline_create_backlight_service, dcpep_cb_create_backlight_service, u8); +/* + * Swap timestamps are in the firmware's continuous time. Firmware 27 counts + * it at 24 MHz, while the ARM counter Linux reads may run faster (1 GHz on + * T6050). A raw counter value then names a moment far in the future and the + * firmware holds the swap for it, so scale the counter to the firmware's + * timebase. + */ +#define DCP_V27_TIMEBASE_HZ 24000000U + +static u64 __maybe_unused dcp_swap_time(void) +{ + u64 now = arch_timer_read_counter(); +#if (DCP_FW_H17(DCP_FW_VER) && DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0)) + u32 rate = arch_timer_get_cntfrq(); + + if (rate && rate != DCP_V27_TIMEBASE_HZ) + return mul_u64_u32_div(now, DCP_V27_TIMEBASE_HZ, rate); +#endif + return now; +} + +/* + * The previous accepted swap's time, or now without one. On firmware 27 also + * now when it is from the future or more than a second old, as after an idle + * period: the firmware paces the swap from it. + */ +static u64 __maybe_unused dcp_swap_prev_time(u64 now, u64 prev) +{ + if (!prev) + return now; +#if (DCP_FW_H17(DCP_FW_VER) && DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0)) + if (prev > now || now - prev > DCP_V27_TIMEBASE_HZ) + return now; +#endif + return prev; +} + /* * Callback for swap requests. If a swap failed, we'll never get a swap * complete event so we need to fake a vblank event early to avoid a hang. @@ -2926,10 +3160,14 @@ static void dcp_swapped(struct apple_dcp *dcp, void *data, void *cookie) return; } dcp->swap_start = ktime_get(); - dcp->swap_submit_timestamp = arch_timer_read_counter(); + dcp->swap_submit_timestamp = dcp_swap_time(); dcp_swap_watchdog_arm(dcp); } +#if (DCP_FW_H17(DCP_FW_VER) && DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0)) +#include "iomfb-peer-runtime-v27.inc" +#endif + static void dcp_swap_started(struct apple_dcp *dcp, void *data, void *cookie) { struct DCP_FW_NAME(dcp_swap_start_resp) *resp = data; @@ -2945,6 +3183,10 @@ static void dcp_swap_started(struct apple_dcp *dcp, void *data, void *cookie) dcp_drm_crtc_vblank(dcp->crtc); return; } +#if (DCP_FW_H17(DCP_FW_VER) && DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0)) + if (dcp_peer_start_second(dcp, resp)) + return; +#endif DCP_FW_SWAP(dcp).swap.swap_id = resp->swap_id; WRITE_ONCE(dcp->started_swap_id, resp->swap_id); if (dcp_present_retires(dcp)) @@ -3528,6 +3770,16 @@ int DCP_FW_NAME(iomfb_modeset)(struct apple_dcp *dcp, static void dcp_on_set_adaptive_sync(struct apple_dcp *dcp, void *out, void *cookie) { +#if (DCP_FW_H17(DCP_FW_VER) && DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0)) + if (!out || get_unaligned_le32(out)) { + struct dcp_wait_cookie *wait = cookie; + + wait->status = -EOPNOTSUPP; + complete(&wait->done); + kref_put(&wait->refcount, release_wait_cookie); + return; + } +#endif dcp_set_digital_out_mode(dcp, false, &dcp->mode, complete_set_digital_out_mode, cookie); } @@ -3536,7 +3788,12 @@ static void dcp_set_adaptive_sync(struct apple_dcp *dcp, u32 min_vrr, void *cookie) { struct dcp_set_parameter_dcp param = { - .param = IOMFBPARAM_ADAPTIVE_SYNC, + .param = +#if (DCP_FW_H17(DCP_FW_VER) && DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0)) + 15, +#else + IOMFBPARAM_ADAPTIVE_SYNC, +#endif .value = { min_vrr, /* minRR, 16.16 fixed-point Hz */ 0, /* mediaTargetRate */ @@ -3619,7 +3876,17 @@ int DCP_FW_NAME(iomfb_modeset)(struct apple_dcp *dcp, DCP_IOMFB_METHODS_H17P_EXTERNAL) { dev_dbg(dcp->dev, "inheriting the bootloader's display mode\n"); complete_set_digital_out_mode(dcp, NULL, cookie); - } else if (mode->vrr && + } else +#if (DCP_FW_H17(DCP_FW_VER) && DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0)) + if ((ret = dcp_peer_modeset_v27(dcp, mode->timing_mode_id, + mode->color_mode_id, crtc_state->vrr_enabled)) != -EOPNOTSUPP) { + cookie->status = ret; + if (!ret) + crtc_state->color_mgmt_changed = true; + complete_set_digital_out_mode(dcp, NULL, cookie); + } else +#endif + if (mode->vrr && dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17P_EXTERNAL) dcp_set_adaptive_sync(dcp, @@ -3853,18 +4120,16 @@ void DCP_FW_NAME(iomfb_flush)(struct apple_dcp *dcp, struct drm_crtc *crtc, stru } #else if (has_surface && (dcp->use_timestamps || dcp->vrr_enabled)) { - u64 submit_timestamp = dcp->swap_submit_timestamp; - u64 timestamp = arch_timer_read_counter(); + u64 timestamp = dcp_swap_time(); + u64 submit_timestamp = + dcp_swap_prev_time(timestamp, dcp->swap_submit_timestamp); /* - * IOMobileFramebuffer uses Mach continuous-time values here. On - * Apple Silicon that is the ARM architectural counter. Empirical + * The firmware takes continuous-time values here, in its own + * timebase (see dcp_swap_time()). Empirical * testing shows that using the current submission time together * with the previous accepted swap makes DCP follow swap pacing. */ - if (!submit_timestamp) - submit_timestamp = timestamp; - /* Firmware 12.x/13.x requires timestamp types 1, 2, and 7. */ req->swap.timestamp[0] = timestamp; req->swap.timestamp[1] = submit_timestamp; @@ -3924,6 +4189,12 @@ void DCP_FW_NAME(iomfb_flush)(struct apple_dcp *dcp, struct drm_crtc *crtc, stru dcp_ctm_to_fw(mat.matrix, crtc_state->ctm ? crtc_state->ctm->data : NULL); +#if (DCP_FW_H17(DCP_FW_VER) && DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0)) + if (dcp_peer_color_v27(dcp, &mat)) { + dcp_peer_swap_error(dcp, "color matrix", -EIO); + return; + } +#endif #if DCP_FW_DESKTOP_26(DCP_FW_VER) dcp->desktop->gamma_pending = dcp_gamma_update(dcp, crtc_state); dcp->desktop->matrix = mat; diff --git a/drivers/gpu/drm/apple/iomfb_v14_7.c b/drivers/gpu/drm/apple/iomfb_v14_7.c index 1f98ace1d9a89..a99bfed67183f 100644 --- a/drivers/gpu/drm/apple/iomfb_v14_7.c +++ b/drivers/gpu/drm/apple/iomfb_v14_7.c @@ -1938,7 +1938,10 @@ static ssize_t dcpext_start_store(struct device *dev, struct device_attribute *a if (atomic_cmpxchg(&dcp->external_requested, 0, 1)) return -EBUSY; /* Keep code and firmware callbacks alive after this one-shot request. */ - __module_get(THIS_MODULE); + if (!try_module_get(THIS_MODULE)) { + atomic_set(&dcp->external_requested, 0); + return -ENODEV; + } schedule_work(&dcp->external_work); return count; } @@ -2066,10 +2069,13 @@ bool iomfb_v14_7_external_ready(struct apple_dcp *dcp) smp_load_acquire(&dcp->external_phase) != DCPEXT_IDLE) return false; if (!atomic_cmpxchg(&dcp->external_requested, 0, 1)) { + /* Firmware callbacks may outlive any later module removal. */ + if (!try_module_get(THIS_MODULE)) { + atomic_set(&dcp->external_requested, 0); + return false; + } WRITE_ONCE(dcp->external_phase, DCPEXT_STARTING); dev_info(dcp->dev, "display attached: starting the external display processor\n"); - /* Firmware callbacks may outlive any later module removal. */ - __module_get(THIS_MODULE); queue_work(system_freezable_wq, &dcp->external_work); } } @@ -2395,17 +2401,23 @@ int iomfb_v14_7_bind(struct apple_dcp *dcp) v14->board->name); dcp_v14_link_init(&v14->link, dev, NULL); + /* The mailbox's callbacks need this module until reboot. */ + if (!try_module_get(THIS_MODULE)) + return dev_err_probe(dev, -ENODEV, + "%s display not started: the module is being removed\n", + v14->board->name); rtk = apple_rtkit_init(dev, v14, "mbox", 0, &dcp_v14_rtkit_ops); - if (IS_ERR(rtk)) + if (IS_ERR(rtk)) { + module_put(THIS_MODULE); return dev_err_probe(dev, PTR_ERR(rtk), "%s display not started: RTKit init failed\n", v14->board->name); + } /* From here nothing is freed and the module stays loaded. */ v14->rtk = rtk; v14->link.rtk = rtk; dcp->rtk = rtk; dcp_v14_panel_session = true; - __module_get(THIS_MODULE); ret = apple_rtkit_wake(rtk); for (n = 0; ret == -ETIME && n < DCP_V14_RTKIT_RETRIES; n++) diff --git a/drivers/gpu/drm/apple/iomfb_v26_6.c b/drivers/gpu/drm/apple/iomfb_v26_6.c index de3cb84dba440..1f456dc17776a 100644 --- a/drivers/gpu/drm/apple/iomfb_v26_6.c +++ b/drivers/gpu/drm/apple/iomfb_v26_6.c @@ -1161,16 +1161,22 @@ int iomfb_v26_6_bind(struct apple_dcp *dcp) return dev_err_probe(dev, -EINVAL, "J613/25G83 same-boot clock observation required\n"); dcp_v26_link_init(&v26->link, dev, NULL); + /* The mailbox's callbacks need this module until reboot. */ + if (!try_module_get(THIS_MODULE)) + return dev_err_probe(dev, -ENODEV, + "%s display not started: the module is being removed\n", + v26->board->name); rtk = apple_rtkit_init(dev, v26, "mbox", 0, &dcp_v26_rtkit_ops); - if (IS_ERR(rtk)) + if (IS_ERR(rtk)) { + module_put(THIS_MODULE); return dev_err_probe(dev, PTR_ERR(rtk), "%s display not started: RTKit init failed\n", v26->board->name); + } /* From here nothing is freed and the module stays loaded. */ v26->rtk = rtk; v26->link.rtk = rtk; dcp_v26_session = true; - __module_get(THIS_MODULE); ret = apple_rtkit_wake(rtk); if (ret) { diff --git a/drivers/gpu/drm/apple/iomfb_v26_6_desktop.c b/drivers/gpu/drm/apple/iomfb_v26_6_desktop.c index 064eff4beff53..c2d0bee01e05f 100644 --- a/drivers/gpu/drm/apple/iomfb_v26_6_desktop.c +++ b/drivers/gpu/drm/apple/iomfb_v26_6_desktop.c @@ -52,46 +52,24 @@ static const struct dcp_method_entry dcp_methods[dcpep_num_methods] = { #include "iomfb_template.c" -struct dcp_dict_passthrough_req_v26_6 { - u8 key[0x40]; - u8 value[0x1000]; - u8 value_null; - u8 padding[3]; -} __packed; - -struct dcp_dict_passthrough_resp_v26_6 { - u8 value[0x1000]; - u8 ret; - u8 padding[3]; -} __packed; - -static_assert(sizeof(struct dcp_dict_passthrough_req_v26_6) == 0x1044); -static_assert(sizeof(struct dcp_dict_passthrough_resp_v26_6) == 0x1004); - /* - * D114 periodically publishes dictionary-valued statistics. The value is an - * in/out pointer, which the firmware deserializes again: the firmware's own - * client returns the serialized buffer unchanged and reports false. An - * all-zero reply is not a harmless false. + * D114 periodically publishes dictionary-valued statistics: the analytics + * event, answered by dcp_dict_passthrough_reply(). */ static bool trampoline_passthrough_dict_v26_6(struct apple_dcp *dcp, int tag, void *out, void *in) { - const struct dcp_dict_passthrough_req_v26_6 *req = in; - struct dcp_dict_passthrough_resp_v26_6 *resp = out; + const struct dcp_dict_passthrough_req *req = in; + struct dcp_dict_passthrough_resp *resp = out; trace_iomfb_callback(dcp, tag, "dcpep_cb_passthrough_dict"); dcp_trace_panel_property(dcp, tag, (const char *)req->key, strnlen((const char *)req->key, sizeof(req->key)), req->value_null ? NULL : req->value, sizeof(req->value)); - if (!req->value_null) { + if (!req->value_null) dcp_panel_edid_receive(dcp, req->value, sizeof(req->value)); - memcpy(resp->value, req->value, sizeof(resp->value)); - } - - resp->ret = false; - memset(resp->padding, 0, sizeof(resp->padding)); + dcp_dict_passthrough_reply(req, resp); return true; } @@ -101,7 +79,7 @@ static bool trampoline_panel_dictionary_v26_6(struct apple_dcp *dcp, int tag, void *out, void *in) { /* Service dictionary setters prefix the same record with a handle. */ - const struct dcp_dict_passthrough_req_v26_6 *req = tag == 413 ? in + 4 : in; + const struct dcp_dict_passthrough_req *req = tag == 413 ? in + 4 : in; const char *key = (const char *)req->key; struct dcp_display_metadata metadata; struct dcp_parse_ctx ctx; @@ -297,8 +275,8 @@ static const struct dcp_cb_min_v26_6_desktop cb_min[IOMFB_MAX_CB] = { [111] = CB_MIN(0, sizeof(u8)), [112] = CB_MIN(0, sizeof(u8)), [113] = CB_MIN(0, sizeof(u8)), - [114] = CB_MIN(sizeof(struct dcp_dict_passthrough_req_v26_6), - sizeof(struct dcp_dict_passthrough_resp_v26_6)), + [114] = CB_MIN(sizeof(struct dcp_dict_passthrough_req), + sizeof(struct dcp_dict_passthrough_resp)), [115] = CB_MIN(sizeof(struct dcpep_get_tiling_state_req), sizeof(struct dcpep_get_tiling_state_resp)), [116] = CB_MIN(sizeof(struct dcpep_get_tiling_state_req), sizeof(u32)), diff --git a/drivers/gpu/drm/apple/iomfb_v27.c b/drivers/gpu/drm/apple/iomfb_v27.c new file mode 100644 index 0000000000000..c6df81bf8abdf --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb_v27.c @@ -0,0 +1,571 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright The Asahi Linux Contributors */ +/* Copyright 2026 Aurora Silicon */ + +#include +#include + +#include "dcp-link.h" +#include "iomfb_v27.h" +#include "iomfb_v12_3.h" +#include "iomfb_v13_3.h" +#include "version_utils.h" +#include "dcp-internal.h" + +void iomfb_serialize_present_v27(struct dcp_swap_submit_req_v27 *wire, + const struct dcp_swap_submit_req_h17p *request) +{ + memset(wire, 0, sizeof(*wire)); + memcpy(wire->swap.timestamp, request->swap.timestamp, + sizeof(wire->swap.timestamp)); + wire->swap.unk_38 = request->swap.unk_38; + /* The common geometry runs from swap_id through bg_color. */ + memcpy(&wire->swap.swap_id, &request->swap.swap_id, + offsetof(struct dcp_swap_v27, reserved_158) - + offsetof(struct dcp_swap_v27, swap_id)); + memcpy(wire->surf, request->surf, sizeof(wire->surf)); + memcpy(wire->surf_iova, request->surf_iova, + sizeof(*request) - + offsetof(struct dcp_swap_submit_req_h17p, surf_iova)); +} + +struct dcp_h17p_hotplug_request { + __le64 connected; + u8 tiled_display[0x4c]; + u8 tiled_display_null; + u8 padding[3]; +} __packed; + +struct dcp_h17p_swap_info_request { + u8 swap_info[0xe0]; + u8 swap_info_null; + u8 padding[3]; +} __packed; + +static_assert(sizeof(struct dcp_h17p_hotplug_request) == 0x58); +static_assert(sizeof(struct dcp_h17p_swap_info_request) == 0xe4); + +static_assert(sizeof(struct dc_swap_complete_resp_h17p) == 0x730); +static_assert(offsetof(struct dc_swap_complete_resp_h17p, swap_id) == 0); +static_assert(offsetof(struct dcp_swap_v27, brightness_update) == 0x354); +static_assert(offsetof(struct dcp_swap_v27, brightness_nits) == 0x35e); +static_assert(sizeof(struct dcp_swap_v27) == 0x6e0); + +static const struct dcp_method_entry dcp_methods[dcpep_num_methods] = { + IOMFB_METHOD("A000", dcpep_late_init_signal), + IOMFB_METHOD("A027", dcpep_setup_video_limits), + IOMFB_METHOD("A136", iomfbep_a131_pmu_service_matched), + IOMFB_METHOD("A137", iomfbep_a132_backlight_service_matched), + IOMFB_METHOD("A389", dcpep_set_create_dfb), + IOMFB_METHOD("A390", iomfbep_a358_vi_set_temperature_hint), + IOMFB_METHOD("A401", dcpep_start_signal), + IOMFB_METHOD("A406", dcpep_swap_start), + IOMFB_METHOD("A408", dcpep_swap_submit), + IOMFB_METHOD("A411", dcpep_set_display_device), + IOMFB_METHOD("A412", dcpep_is_main_display), + IOMFB_METHOD("A413", dcpep_set_digital_out_mode), + IOMFB_METHOD("A423", iomfbep_set_matrix), + IOMFB_METHOD("A427", iomfbep_get_color_remap_mode), + IOMFB_METHOD("A450", dcpep_set_parameter_dcp), + IOMFB_METHOD("A454", dcpep_create_default_fb), + IOMFB_METHOD("A037", dcpep_get_dfb_compression_info), + IOMFB_METHOD("A036", dcpep_get_dfb_info), + IOMFB_METHOD("A103", dcpep_get_dfb_layout), + IOMFB_METHOD("A453", dcpep_get_dfb_state), + IOMFB_METHOD("A104", dcpep_set_dfb_dimensions), + IOMFB_METHOD("A105", dcpep_commit_dfb_info), + IOMFB_METHOD("A035", dcpep_dfb_query), + IOMFB_METHOD("A384", dcpep_dfb_ready_query), + IOMFB_METHOD("A353", dcpep_pipe_query_353), + IOMFB_METHOD("A352", dcpep_pipe_set_352), + IOMFB_METHOD("A415", dcpep_pipe_cfg_415), + IOMFB_METHOD("A033", dcpep_pipe_cfg_031), + IOMFB_METHOD("A414", dcpep_pipe_cfg_414), + IOMFB_METHOD("A489", dcpep_pipe_query_478), + IOMFB_METHOD("A485", dcpep_pipe_query_474), + IOMFB_METHOD("A487", dcpep_pipe_set_476), + IOMFB_METHOD("A428", iomfbep_set_brightness_correction), + IOMFB_METHOD("A458", dcpep_enable_disable_video_power_savings), + IOMFB_METHOD("A466", dcpep_first_client_open), + IOMFB_METHOD("A467", iomfbep_last_client_close), + IOMFB_METHOD("A475", dcpep_set_display_refresh_properties), + IOMFB_METHOD("A478", dcpep_flush_supports_power), + IOMFB_METHOD("A479", iomfbep_abort_swaps_dcp), + IOMFB_METHOD("A484", dcpep_set_power_state), + IOMFB_METHOD("A483", dcpep_register_dfb_surface), +}; + +#define iomfb_modeset_h17p iomfb_modeset_v27 +#define iomfb_flush_h17p iomfb_flush_v27 +#define iomfb_poweron_h17p iomfb_poweron_v27 +#define iomfb_poweroff_h17p iomfb_poweroff_v27 +#define iomfb_sleep_h17p iomfb_sleep_v27 +#define iomfb_shutdown_h17p iomfb_shutdown_v27 +#define iomfb_present_backlight_h17p iomfb_present_backlight_v27 +#define DCP_FW h17p +#define DCP_FW_VER DCP_FW_VERSION(27, 0, 0) + +#include "iomfb_template.c" + +static bool trampoline_rt_bandwidth_h17p(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + u64 scratch = dcp->disp_firmware_base[dcp->disp_bw_scratch_index] + + dcp->disp_bw_scratch_offset; + u64 clock_request = 0; + + trace_iomfb_callback(dcp, tag, __func__); + if (apple_dcp_h17p_rt_bw_encode_reply(in, out, scratch, clock_request)) + dev_warn(dcp->dev, "D003 bandwidth request is malformed\n"); + return true; +} + +static bool trampoline_get_frequency_v27(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + const struct apple_dcp_h17p_clock_request *request = in; + u32 index = le32_to_cpu(request->clock_id); + u64 rate = 0; + + if (index < dcp->num_firmware_clocks) + rate = clk_get_rate(dcp->firmware_clocks[index].clk); + else + dev_err(dcp->dev, "DCP requested undeclared provider clock index %u\n", + index); + *(__le64 *)out = cpu_to_le64(rate); + return true; +} + +static bool trampoline_hotplug_h17p(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + const struct dcp_h17p_hotplug_request *request = in; + + /* The opt-in peer allocator needs the firmware's tiling metadata to verify + * the VFTG relationship before implementing its framebuffer lifecycle. + * Keep both halves: the endpoint IDs are at the end of this payload. + */ + if (dcp->peer_reserved || dcp->peer_owner) { + dev_info(dcp->dev, "peer HPD connected=%llu null=%u head=%*ph\n", + le64_to_cpu(request->connected), request->tiled_display_null, + 40, request->tiled_display); + dev_info(dcp->dev, "peer HPD tail=%*ph\n", 36, + request->tiled_display + 40); + } + if (!(request->tiled_display_null & 1) && request->connected && + !(get_unaligned_le32(request->tiled_display + 0xc) & BIT(3))) + dcp_peer_display_ids_v27(dcp, + get_unaligned_le32(request->tiled_display + 0x3b), + get_unaligned_le32(request->tiled_display + 0x47)); + if (!(request->tiled_display_null & 1)) + memcpy(out, request->tiled_display, + sizeof(request->tiled_display)); + return trampoline_hotplug(dcp, tag, out, in); +} + +static bool trampoline_swap_info_h17p(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + const struct dcp_h17p_swap_info_request *request = in; + + trace_iomfb_callback(dcp, tag, __func__); + if (!(request->swap_info_null & 1)) + memcpy(out, request->swap_info, sizeof(request->swap_info)); + return true; +} + +static bool iomfbep_cb_video_range_h17p(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + __le32 *resp = out; + + resp[0] = cpu_to_le32(0x435b0000); + resp[1] = 0; + return true; +} + +struct dcp_h17p_get_data_prop_req { + char obj[4]; + char key[0x40]; + __le32 length; + u8 data_null; + u8 length_null; + u8 padding[2]; +} __packed; +static_assert(sizeof(struct dcp_h17p_get_data_prop_req) == 0x4c); + +struct dcp_h17p_get_data_prop_resp { + u8 data[0xc00]; + __le32 length; +} __packed; +static_assert(sizeof(struct dcp_h17p_get_data_prop_resp) == 0xc04); + +static bool trampoline_get_data_prop_v27(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + const struct dcp_h17p_get_data_prop_req *req = in; + struct dcp_h17p_get_data_prop_resp *resp = out; + const struct property *prop; + u32 cap = min_t(u32, le32_to_cpu(req->length), sizeof(resp->data)); + + if (memcmp(req->obj, "VORP", sizeof(req->obj)) || + strnlen(req->key, sizeof(req->key)) == sizeof(req->key)) + return true; + + prop = of_find_property(dcp->dev->of_node, req->key, NULL); + if (!prop || prop->length < 0 || prop->length > cap) + return true; + memcpy(resp->data, prop->value, prop->length); + resp->length = cpu_to_le32(prop->length); + return true; +} + +struct dcp_allocate_buffer_req_v27 { + __le32 options; + __le64 size; + __le32 alignment; + u8 dva_null; + u8 capacity_null; + u8 padding[2]; +} __packed; + +struct dcp_allocate_buffer_resp_v27 { + __le64 dva; + __le64 capacity; + __le32 mem_desc_id; +} __packed; + +struct dcp_buffer_lifecycle_req_v27 { + __le32 mem_desc_id; + __le32 options; +} __packed; + +static_assert(sizeof(struct dcp_allocate_buffer_req_v27) == 0x14); +static_assert(sizeof(struct dcp_allocate_buffer_resp_v27) == 0x14); +static_assert(offsetof(struct dcp_allocate_buffer_resp_v27, mem_desc_id) == 0x10); +static_assert(sizeof(struct dcp_buffer_lifecycle_req_v27) == 8); + +static bool trampoline_allocate_buffer_v27(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + const struct dcp_allocate_buffer_req_v27 *req = in; + struct dcp_allocate_buffer_resp_v27 *resp = out; + struct dcp_allocate_buffer_req allocation = { + .unk0 = le32_to_cpu(req->options), + .size = le64_to_cpu(req->size), + .unk2 = le32_to_cpu(req->alignment), + }; + struct dcp_allocate_buffer_resp buffer; + + if (!allocation.size || allocation.size > SIZE_MAX - SZ_16K || + allocation.unk2 > PAGE_SIZE || + (allocation.unk2 && !is_power_of_2(allocation.unk2))) + return true; + buffer = dcpep_cb_allocate_buffer(dcp, &allocation); + if (!req->dva_null) + resp->dva = cpu_to_le64(buffer.dva); + if (!req->capacity_null) + resp->capacity = cpu_to_le64(ALIGN(buffer.dva_size, SZ_16K)); + resp->mem_desc_id = cpu_to_le32(buffer.mem_desc_id); + return true; +} + +static bool trampoline_buffer_lifecycle_v27(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + const struct dcp_buffer_lifecycle_req_v27 *req = in; + __le32 *status = out; + u32 id = le32_to_cpu(req->mem_desc_id); + + if (!id || id >= DCP_MAX_MAPPINGS || + !test_bit(id, dcp->memdesc_map) || !dcp->memdesc[id].buf) { + *status = cpu_to_le32(14); + return true; + } + if (tag == 452) + dma_wmb(); + else + dma_rmb(); + *status = 0; + return true; +} + +static bool trampoline_ideal_screen_space_v27(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + const char *gamut; + u32 space = 1; + + if (!of_property_read_string(dcp->dev->of_node, + "apple,artwork-display-gamut", &gamut)) { + if (!strcmp(gamut, "P3")) + space = 2; + else if (!strcmp(gamut, "AdobeRGB")) + space = 3; + } + *(__le32 *)out = cpu_to_le32(space); + return true; +} + +static bool trampoline_tiling_state_v27(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + const struct dcpep_get_tiling_state_req *req = in; + struct dcpep_get_tiling_state_resp *resp = out; + + resp->value = req->value; + resp->ret = 0; + if (!(req->value_null & 1) && + dcp_peer_tiling_state_v27(dcp, false, req->event, req->param, + &resp->value)) { + resp->ret = 1; + return true; + } + if (!req->value_null && (req->param == 1 || req->param == 11) && + dcp->connector && !dcp->connector->base.has_tile) { + resp->value = 0; + resp->ret = 1; + } + return true; +} + +/* D115: same16-byte input as D114, boolean/padded4 output. */ +static bool trampoline_set_tiling_state_v27(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + const struct dcpep_get_tiling_state_req *req = in; + u32 value = req->value; + + *(__le32 *)out = cpu_to_le32(!(req->value_null & 1) && + dcp_peer_tiling_state_v27(dcp, true, req->event, req->param, &value)); + return true; +} + +#include "iomfb-display-identity-v27.inc" +#include "iomfb-external-counts-v27.inc" +#include "iomfb-boolean-properties-v27.inc" + +#define DCP_ANALYTICS_EVENT_SIZE 0x1044 +#define DCP_ANALYTICS_NAME_SIZE 0x40 +#define DCP_ANALYTICS_DICT_SIZE (DCP_ANALYTICS_EVENT_SIZE - DCP_ANALYTICS_NAME_SIZE) +#define DCP_ANALYTICS_FULL_DUMPS 16 + +/* + * D113: the firmware reports an analytics event, its name in the first 0x40 + * bytes and then the event's serialized payload dictionary. The dictionary is + * an in/out argument: the 0x1004-byte reply area is the dictionary the + * firmware reads back once the call returns. Hand it back unchanged. + * + * Replying with zeroes there made every DCP crash on its first analytics + * event, which the firmware sends about an hour after it starts; all DCPs + * start together at boot, so all of them went down within seconds. + */ +static bool dcpep_cb_core_analytics_v27(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + const u8 *event = in; + unsigned int len = DCP_ANALYTICS_EVENT_SIZE; + char name[128]; + + static_assert(DCP_ANALYTICS_DICT_SIZE == 0x1004); + + strscpy(name, event, min(sizeof(name), (size_t)DCP_ANALYTICS_NAME_SIZE)); + dev_info_ratelimited(dcp->dev, "CoreAnalytics event %s, reply area %16ph\n", + name, out); + if (dcp->analytics_dumps < DCP_ANALYTICS_FULL_DUMPS) { + dcp->analytics_dumps++; + while (len && !event[len - 1]) + len--; + print_hex_dump(KERN_INFO, "dcp analytics: ", DUMP_PREFIX_OFFSET, + 32, 1, event, len, true); + } + /* The same record and answer as desktop 26.6's analytics event. */ + dcp_dict_passthrough_reply(in, out); + return true; +} + +static const iomfb_cb_handler cb_handlers[IOMFB_MAX_CB] = { + [0] = dcpep_cb_d000_h17p, + [1] = trampoline_true, + [2] = trampoline_nop, + [3] = trampoline_rt_bandwidth_h17p, + [6] = trampoline_set_frame_sync_props_h17p, + [100] = iomfbep_cb_match_pmu_service, + [101] = trampoline_zero, + [102] = trampoline_nop, + [104] = trampoline_nop, + [106] = trampoline_nop, + [107] = trampoline_true, + [108] = trampoline_true, + [109] = trampoline_true, + [110] = trampoline_true, + [111] = trampoline_create_backlight_service, + [112] = trampoline_true, + [113] = dcpep_cb_core_analytics_v27, + [116] = trampoline_nop, + [117] = trampoline_zero, + [118] = trampoline_zero, + [119] = trampoline_zero, + [114] = trampoline_tiling_state_v27, + [115] = trampoline_set_tiling_state_v27, + [120] = dcpep_cb_boot_1, + [121] = trampoline_false, + [122] = trampoline_false, + [123] = trampoline_zero, + [124] = trampoline_read_edt_data, + [126] = trampoline_prop_start, + [127] = trampoline_prop_chunk, + [128] = trampoline_prop_end, + [129] = trampoline_allocate_bandwidth, + [130] = trampoline_update_external_counts_v27, + [131] = trampoline_get_external_counts_v27, + [132] = trampoline_apply_suppression_v27, + [133] = trampoline_display_identity_v27, + [201] = trampoline_map_piodma, + [202] = trampoline_unmap_piodma, + [206] = iomfbep_cb_match_pmu_service_2, + [207] = iomfbep_cb_match_backlight_service, + [204] = trampoline_ideal_screen_space_v27, + [208] = trampoline_nop, + [209] = trampoline_get_time, + [300] = trampoline_pr_publish, + [400] = trampoline_get_data_prop_v27, + [401] = trampoline_get_uint_prop, + [404] = trampoline_nop, + [406] = trampoline_set_fx_prop, + [408] = trampoline_get_frequency_v27, + [411] = trampoline_map_reg, + [413] = trampoline_true, + [414] = trampoline_sr_set_property_int, + [415] = trampoline_true, + [451] = trampoline_allocate_buffer_v27, + [452] = trampoline_buffer_lifecycle_v27, + [453] = trampoline_buffer_lifecycle_v27, + [454] = trampoline_release_mem_desc, + [552] = trampoline_true, + [561] = trampoline_true, + [563] = trampoline_true, + [565] = trampoline_true, + [567] = trampoline_true, + [569] = trampoline_set_boolean_property_v27, + [572] = trampoline_zero, + [574] = iomfbep_cb_video_range_h17p, + [575] = trampoline_hotplug_h17p, + [576] = trampoline_nop, + [586] = iomfbep_cb_create_dfb_surface, + [587] = trampoline_nop, + [588] = trampoline_nop, + [589] = trampoline_swap_info_h17p, + + [592] = trampoline_nop, + [594] = trampoline_swap_complete, + [596] = trampoline_swap_complete_intent_gated, + [597] = trampoline_abort_swap_ap_gated, + [598] = trampoline_enable_backlight_message_ap_gated, + [599] = trampoline_nop, + [601] = trampoline_false, + [602] = trampoline_false, + [603] = trampoline_nop, +}; + +static const struct { + u16 input, output; +} callback_sizes[IOMFB_MAX_CB] = { + [0] = { 0x0, 0x4 }, + [1] = { 0x0, 0x4 }, + [2] = { 0x0, 0x0 }, + [3] = { 0x4, 0x14 }, + [6] = { 0x54, 0x50 }, + [300] = { 0x10, 0x0 }, + [100] = { 0x0, 0x0 }, + [101] = { 0x0, 0x4 }, + [102] = { 0x44, 0x0 }, + [104] = { 0x44, 0x0 }, + [106] = { 0x40, 0x0 }, + [107] = { 0x0, 0x4 }, + [108] = { 0x0, 0x4 }, + [109] = { 0x0, 0x4 }, + [110] = { 0x0, 0x4 }, + [111] = { 0x0, 0x4 }, + [112] = { 0x0, 0x4 }, + [113] = { 0x1044, 0x1004 }, + [116] = { 0xc, 0x0 }, + [117] = { 0x14, 0x4 }, + [118] = { 0x3e88, 0x4 }, + [119] = { 0x4, 0x4 }, + [114] = { 0x10, 0x8 }, + [115] = { 0x10, 0x4 }, + [120] = { 0x0, 0x4 }, + [121] = { 0x0, 0x4 }, + [122] = { 0x0, 0x4 }, + [123] = { 0x0, 0x4 }, + [124] = { 0x64, 0x24 }, + [126] = { 0x4, 0x4 }, + [127] = { 0x1008, 0x4 }, + [128] = { 0x40, 0x4 }, + [129] = { 0x1c, 0x14 }, + [130] = { 0x8, 0x0 }, + [131] = { 0x10, 0xc }, + [132] = { 0x4, 0x0 }, + [133] = { 0x0, 0x4 }, + [201] = { 0xc, 0x10 }, + [202] = { 0x18, 0x0 }, + [206] = { 0x0, 0x4 }, + [207] = { 0x0, 0x4 }, + [204] = { 0x0, 0x4 }, + [208] = { 0x4, 0x0 }, + [209] = { 0x0, 0x8 }, + [400] = { 0x4c, 0xc04 }, + [401] = { 0x50, 0xc }, + [404] = { 0x48, 0x0 }, + [406] = { 0x48, 0x0 }, + [408] = { 0x8, 0x8 }, + [411] = { 0x10, 0x1c }, + [413] = { 0x1048, 0x4 }, + [414] = { 0x50, 0x4 }, + [415] = { 0x4c, 0x4 }, + [451] = { 0x14, 0x14 }, + [452] = { 0x8, 0x4 }, + [453] = { 0x8, 0x4 }, + [454] = { 0x4, 0x4 }, + [552] = { 0x1044, 0x4 }, + [561] = { 0x1044, 0x4 }, + [563] = { 0x4c, 0x4 }, + [565] = { 0x48, 0x4 }, + [567] = { 0x80, 0x4 }, + [569] = { 0x44, 0x4 }, + [572] = { 0x4, 0x4 }, + [574] = { 0x4, 0x8 }, + [575] = { 0x58, 0x4c }, + [576] = { 0x4, 0x0 }, + [586] = { 0x8, 0x4 }, + [587] = { 0x0, 0x0 }, + [588] = { 0x18, 0x0 }, + [589] = { 0xe4, 0xe0 }, + [592] = { 0x0, 0x0 }, + [594] = { 0x730, 0x0 }, + [596] = { 0x14, 0x0 }, + [597] = { 0x4, 0x0 }, + [598] = { 0x4, 0x0 }, + [599] = { 0x4, 0x0 }, + [601] = { 0x0, 0x4 }, + [602] = { 0x0, 0x4 }, + [603] = { 0x0, 0x0 }, +}; + +bool iomfb_v27_callback_size_valid(unsigned int tag, u32 input, u32 output) +{ + if (tag >= ARRAY_SIZE(callback_sizes) || !cb_handlers[tag]) + return false; + return input >= callback_sizes[tag].input && + output >= callback_sizes[tag].output; +} + +void iomfb_start_v27(struct apple_dcp *dcp) +{ + dcp->cb_handlers = cb_handlers; + dcp_start_signal(dcp, false, dcp_started, NULL); +} + +#undef DCP_FW_VER +#undef DCP_FW diff --git a/drivers/gpu/drm/apple/iomfb_v27.h b/drivers/gpu/drm/apple/iomfb_v27.h new file mode 100644 index 0000000000000..ac1ded5eb047c --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb_v27.h @@ -0,0 +1,95 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright The Asahi Linux Contributors */ +/* Copyright 2026 Aurora Silicon */ + +#ifndef APPLE_IOMFB_V27_H +#define APPLE_IOMFB_V27_H + +#include "iomfb_h17p.h" + +struct dcp_swap_v27 { + + u64 timestamp[7]; + u64 unk_38; + + u64 debug_info[2]; + + u64 event_wait[SWAP_SURFACES]; + u64 event_signal_on_glass; + u64 event_signal[SWAP_SURFACES]; + + u32 swap_id; + + u32 surf_ids[SWAP_SURFACES]; + struct dcp_rect src_rect[SWAP_SURFACES]; + u32 surf_flags[SWAP_SURFACES]; + u32 surf_unk[SWAP_SURFACES]; + struct dcp_rect dst_rect[SWAP_SURFACES]; + u32 swap_enabled; + u32 swap_completed; + + u32 bg_color; + u8 reserved_158[0x1fc]; + + u8 brightness_update; + u8 reserved_355[9]; + u64 brightness_nits; + u8 reserved_366[0x37a]; +} __packed; + +struct dcp_surface_v27 { + struct dcp_surface base; + u8 padding[47]; +} __packed; + +struct dcp_swap_submit_req_v27 { + struct dcp_swap_v27 swap; + struct dcp_surface_v27 surf[SWAP_SURFACES]; + u64 surf_iova[SWAP_SURFACES]; + u64 unk_u64_a[SWAP_SURFACES]; + + u8 unkbool; + u64 unkdouble; + u64 unkU64; + u8 unkbool2; + u32 clear; + u32 unkU32Ptr; + u8 swap_null; + u8 surf_null[SWAP_SURFACES]; + u8 unkoutbool_null; + u8 unknown_pointer_null; + u8 unknown_output_null; + u8 padding[2]; +} __packed; + +struct dcp_swap_submit_resp_v27 { + u8 unkoutbool; + u32 unkU32out; + u32 ret; + u8 padding[3]; +} __packed; + +static_assert(sizeof(struct dcp_swap_v27) == 0x6e0); +static_assert(sizeof(struct dcp_surface_v27) == 0x22c); +static_assert(sizeof(struct dcp_swap_submit_req_v27) == 0xff4); +static_assert(offsetof(struct dcp_swap_submit_req_v27, surf) == 0x6e0); +static_assert(offsetof(struct dcp_swap_submit_req_v27, surf_iova) == 0xf90); +static_assert(offsetof(struct dcp_swap_submit_req_v27, swap_null) == 0xfea); +static_assert(offsetof(struct dcp_swap_v27, brightness_update) == 0x354); +static_assert(offsetof(struct dcp_swap_v27, brightness_nits) == 0x35e); + +void iomfb_serialize_present_v27(struct dcp_swap_submit_req_v27 *wire, + const struct dcp_swap_submit_req_h17p *request); +bool iomfb_v27_callback_size_valid(unsigned int tag, u32 input, u32 output); +void iomfb_firmware_state_cleanup_v27(void *data); +int iomfb_modeset_v27(struct apple_dcp *dcp, struct drm_crtc_state *state); +void iomfb_flush_v27(struct apple_dcp *dcp, struct drm_crtc *crtc, + struct drm_atomic_commit *commit); +void iomfb_start_v27(struct apple_dcp *dcp); +void iomfb_shutdown_v27(struct apple_dcp *dcp); +void iomfb_poweron_v27(struct apple_dcp *dcp); +void iomfb_poweroff_v27(struct apple_dcp *dcp); +void iomfb_sleep_v27(struct apple_dcp *dcp); +void iomfb_present_backlight_v27(struct apple_dcp *dcp); + +#endif diff --git a/drivers/gpu/drm/apple/parser.c b/drivers/gpu/drm/apple/parser.c index 6e48211d69f53..ca95e4f71eead 100644 --- a/drivers/gpu/drm/apple/parser.c +++ b/drivers/gpu/drm/apple/parser.c @@ -15,6 +15,7 @@ #include "parser.h" #include "dcp-refresh.h" +#include "dcp-peer-v27.h" #include "trace.h" #define DCP_PARSE_HEADER 0xd3 @@ -1761,12 +1762,58 @@ int parse_panel_metadata(struct dcp_parse_ctx *handle, struct dcp_display_metada return 0; } +int parse_dptx_caps(struct dcp_parse_ctx *handle, struct dcp_dptx_caps *caps) +{ + struct iterator it; + struct dcp_dptx_caps parsed = {}; + int ret = 0; + + ret = iterator_begin(handle, &it, true); + if (ret) + return ret; + for (; it.idx < it.len; ++it.idx) { + u32 *field = NULL; + char *key = parse_string(it.handle); + s64 value; + + if (IS_ERR(key)) + return PTR_ERR(key); + if (!strcmp(key, "MaxPixelWidth")) + field = &parsed.width; + else if (!strcmp(key, "MaxTotalPixelRate")) + field = &parsed.total_rate; + else if (!strcmp(key, "MaxActivePixelRate")) + field = &parsed.active_rate; + kfree(key); + if (!field) { + ret = skip(it.handle); + if (ret) + return ret; + if (it.handle->pos > it.handle->len) + return -EINVAL; + continue; + } + if (*field) + return -EINVAL; + ret = parse_int_bound(it.handle, &value, 1, U32_MAX); + if (ret) + return ret; + *field = value; + } + if (!parsed.width || !parsed.total_rate || !parsed.active_rate) + return -ENOENT; + *caps = parsed; + return 0; +} + int parse_epic_service_init(struct dcp_parse_ctx *handle, const char **name, - const char **class, s64 *unit, bool h17p_keys) + const char **class, s64 *unit, bool h17p_keys, + bool epic_unit_first) { int ret = 0; struct iterator it; bool parsed_unit = false; + bool explicit_unit = false; bool parsed_name = false; bool parsed_class = false; @@ -1793,7 +1840,9 @@ int parse_epic_service_init(struct dcp_parse_ctx *handle, const char **name, * service to ask and idles the panel, so no vblank and no swap * ever completes. Only H17P callers accept them. If a * dictionary carries both spellings, the later value wins and - * the earlier string is freed. + * the earlier string is freed, except that with + * @epic_unit_first (firmware 27) EPICUnit is the unit in + * either order. */ if (!strcmp(key, "EPICName") || (h17p_keys && !strcmp(key, "compartment"))) { @@ -1814,10 +1863,13 @@ int parse_epic_service_init(struct dcp_parse_ctx *handle, const char **name, else parsed_class = true; } else if (!strcmp(key, "EPICUnit") || - (h17p_keys && !strcmp(key, "interface-id"))) { + (h17p_keys && !explicit_unit && !strcmp(key, "interface-id"))) { ret = parse_int(it.handle, unit); - if (!ret) + if (!ret) { parsed_unit = true; + explicit_unit |= epic_unit_first && + !strcmp(key, "EPICUnit"); + } } else { ret = skip(it.handle); } diff --git a/drivers/gpu/drm/apple/parser.h b/drivers/gpu/drm/apple/parser.h index 3fa384ecf68c4..45aaa1739ea93 100644 --- a/drivers/gpu/drm/apple/parser.h +++ b/drivers/gpu/drm/apple/parser.h @@ -17,6 +17,7 @@ typedef void (*dcp_identity_key_fn)(void *data, const char *key, u32 key_len, int parse_identity_keys(const void *blob, size_t len, dcp_identity_key_fn emit, void *data); struct apple_connector; struct apple_dcp_typec_route; +struct dcp_dptx_caps; struct dcp_parse_ctx { struct apple_dcp *dcp; @@ -224,7 +225,9 @@ int parse_panel_product_metadata(struct dcp_parse_ctx *handle, int parse_display_attributes(struct dcp_parse_ctx *handle, int *width_mm, int *height_mm, bool *backlight_control); int parse_epic_service_init(struct dcp_parse_ctx *handle, const char **name, - const char **class, s64 *unit, bool h17p_keys); + const char **class, s64 *unit, bool h17p_keys, + bool epic_unit_first); +int parse_dptx_caps(struct dcp_parse_ctx *handle, struct dcp_dptx_caps *caps); struct dcp_sound_format_mask { u64 formats; /* SNDRV_PCM_FMTBIT_* */ diff --git a/drivers/gpu/drm/apple/parser_test.c b/drivers/gpu/drm/apple/parser_test.c index 7fc03175a2324..ed3f99f6a9ddd 100644 --- a/drivers/gpu/drm/apple/parser_test.c +++ b/drivers/gpu/drm/apple/parser_test.c @@ -56,13 +56,20 @@ static void blob_int(struct blob *b, s64 value) b->len += 8; } -static int parse_service(struct kunit *test, const struct blob *b, bool h17p, - const char **name, const char **class, s64 *unit) +static int parse_service_v27(struct kunit *test, const struct blob *b, bool h17p, + bool v27, const char **name, const char **class, + s64 *unit) { struct dcp_parse_ctx ctx; KUNIT_ASSERT_EQ(test, parse(b->data, b->len, &ctx), 0); - return parse_epic_service_init(&ctx, name, class, unit, h17p); + return parse_epic_service_init(&ctx, name, class, unit, h17p, v27); +} + +static int parse_service(struct kunit *test, const struct blob *b, bool h17p, + const char **name, const char **class, s64 *unit) +{ + return parse_service_v27(test, b, h17p, false, name, class, unit); } static void parser_service_init_test(struct kunit *test) @@ -118,6 +125,36 @@ static void parser_service_init_h17p_keys_test(struct kunit *test) kfree(class); } +/* Firmware 27 takes EPICUnit over interface-id; H17P keeps the later key. */ +static void parser_service_init_v27_unit_test(struct kunit *test) +{ + const char *name, *class; + struct blob b; + s64 unit = -1; + + blob_init(&b); + blob_tag(&b, T_DICT, 4); + blob_string(&b, "EPICName"); + blob_string(&b, "dcpdptx-port-epic"); + blob_string(&b, "EPICProviderClass"); + blob_string(&b, "AppleDCPDPTXRemotePort"); + blob_string(&b, "EPICUnit"); + blob_int(&b, 1); + blob_string(&b, "interface-id"); + blob_int(&b, 3); + + KUNIT_ASSERT_EQ(test, parse_service_v27(test, &b, true, true, &name, &class, + &unit), 0); + KUNIT_EXPECT_EQ(test, unit, 1); + kfree(name); + kfree(class); + + KUNIT_ASSERT_EQ(test, parse_service(test, &b, true, &name, &class, &unit), 0); + KUNIT_EXPECT_EQ(test, unit, 3); + kfree(name); + kfree(class); +} + static void parser_service_init_truncated_test(struct kunit *test) { const char *name, *class; @@ -271,6 +308,7 @@ static void parser_property_complete_test(struct kunit *test) static struct kunit_case parser_cases[] = { KUNIT_CASE(parser_service_init_test), KUNIT_CASE(parser_service_init_h17p_keys_test), + KUNIT_CASE(parser_service_init_v27_unit_test), KUNIT_CASE(parser_service_init_truncated_test), KUNIT_CASE(parser_nesting_test), KUNIT_CASE(parser_bad_header_test), diff --git a/drivers/gpu/drm/apple/plane.c b/drivers/gpu/drm/apple/plane.c index a8c37e29ed65a..bf25e66a9322a 100644 --- a/drivers/gpu/drm/apple/plane.c +++ b/drivers/gpu/drm/apple/plane.c @@ -145,8 +145,16 @@ static int apple_get_ix_layout(u32 width, u32 height, return 0; } +static bool apple_plane_element_geometry(struct drm_crtc *crtc) +{ + struct apple_dcp *dcp = platform_get_drvdata(to_apple_crtc(crtc)->dcp); + + return dcp_surface_element_geometry(dcp); +} + /* Use exactly the same geometry for admission and surface construction. */ static int apple_plane_get_fb_layout(struct drm_framebuffer *fb, + bool element_geometry, struct apple_plane_layout *planes, u32 *buf_size) { @@ -195,7 +203,9 @@ static int apple_plane_get_fb_layout(struct drm_framebuffer *fb, return -EINVAL; p->image.tile_w = drm_format_info_block_width(fmt, i); p->image.tile_h = drm_format_info_block_height(fmt, i); - p->tile_size = p->image.tile_w * p->image.tile_h; + /* tile_size is bytes per element, not pixels. */ + p->tile_size = element_geometry ? fmt->char_per_block[i] : + p->image.tile_w * p->image.tile_h; p->stride = fb->pitches[i]; break; default: @@ -251,7 +261,9 @@ static int apple_plane_atomic_check(struct drm_plane *plane, if (ret < 0) return ret; - ret = apple_plane_get_fb_layout(new_plane_state->fb, layouts, &buf_size); + ret = apple_plane_get_fb_layout(new_plane_state->fb, + apple_plane_element_geometry(new_plane_state->crtc), + layouts, &buf_size); if (ret) return ret; @@ -417,8 +429,9 @@ static void apple_plane_atomic_update(struct drm_plane *plane, struct drm_framebuffer *fb = base->fb; const struct drm_format_info *fmt = fb->format; + bool element_geometry = apple_plane_element_geometry(base->crtc); - if (WARN_ON_ONCE(apple_plane_get_fb_layout(fb, layouts, &buf_size))) { + if (WARN_ON_ONCE(apple_plane_get_fb_layout(fb, element_geometry, layouts, &buf_size))) { memset(&new_state->surf, 0, sizeof(new_state->surf)); return; } @@ -450,7 +463,7 @@ static void apple_plane_atomic_update(struct drm_plane *plane, .surface_id = plane->base.id, .buf_size = buf_size, - /* Only used for compressed or multiplanar surfaces */ + /* Only used for compressed or multiplanar surfaces before firmware 27 */ .pix_size = 1, .pel_w = 1, .pel_h = 1, @@ -459,6 +472,16 @@ static void apple_plane_atomic_update(struct drm_plane *plane, .has_compr_info = 1, }; + if (element_geometry) { + /* Bytes per element, and the element's width and height in pixels */ + new_state->surf.pix_size = fmt->char_per_block[0]; + new_state->surf.pel_w = drm_format_info_block_width(fmt, 0); + new_state->surf.pel_h = drm_format_info_block_height(fmt, 0); + /* Present only when compression_info describes a compressed plane */ + new_state->surf.has_compr_info = + fb->modifier == DRM_FORMAT_MOD_APPLE_INTERCHANGE; + } + /* Populate plane information for planar formats */ struct dcp_surface *surf = &new_state->surf; for (int i = 0; fb->format->num_planes && i < fb->format->num_planes; i++) { diff --git a/drivers/gpu/drm/apple/systemep.c b/drivers/gpu/drm/apple/systemep.c index 695b8b402ab98..8b18d4cd2b37e 100644 --- a/drivers/gpu/drm/apple/systemep.c +++ b/drivers/gpu/drm/apple/systemep.c @@ -108,7 +108,7 @@ static int powerlog_report(struct apple_epic_service *service, enum epic_subtype dev_dbg(dcp->dev, "systemep: mNits event: Nits: %u.%03u, iDAC: %u\n", mnits.millinits / 1000, mnits.millinits % 1000, mnits.idac); have_mnits: - if ((dcp->fw_compat == DCP_FIRMWARE_H17P && + if (((dcp->fw_compat == DCP_FIRMWARE_H17P || dcp->fw_compat == DCP_FIRMWARE_V_27_0) && dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) || dcp->fw_compat == DCP_FIRMWARE_V_26_6_DESKTOP) { if (!dcp_has_panel(dcp)) diff --git a/drivers/gpu/drm/apple/version_utils.h b/drivers/gpu/drm/apple/version_utils.h index 8851ee57c2daf..8cf36e39d3f24 100644 --- a/drivers/gpu/drm/apple/version_utils.h +++ b/drivers/gpu/drm/apple/version_utils.h @@ -61,6 +61,7 @@ enum dcp_firmware_version { * (iomfb_v26_6_desktop.c); not the J613 DCPLink DCP_FIRMWARE_V_26_6. */ DCP_FIRMWARE_V_26_6_DESKTOP, + DCP_FIRMWARE_V_27_0, }; #endif /*__APPLE_VERSION_UTILS_H__*/ diff --git a/drivers/gpu/drm/asahi/Makefile b/drivers/gpu/drm/asahi/Makefile index db13d06336b3c..e50a4d768369f 100644 --- a/drivers/gpu/drm/asahi/Makefile +++ b/drivers/gpu/drm/asahi/Makefile @@ -1,9 +1,11 @@ # SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_DRM_ASAHI) += aurora.o -aurora-y := asahi_core.o fwid.o sysfs.o g17/sync_input.o g17/sync_output.o g17/sync_scope.o g17/trace.o g17/clpc.o +aurora-y := asahi_core.o fwid.o sysfs.o g17/sync_input.o g17/sync_output.o g17/sync_scope.o g17/trace.o g17/clpc.o g17/fans.o CFLAGS_g17/trace.o := -I$(src)/g17 aurora-$(CONFIG_DRM_ASAHI_G15) += g15/glue.o aurora-$(CONFIG_DRM_ASAHI_G16) += g16/glue.o + +aurora-y += retain.o diff --git a/drivers/gpu/drm/asahi/asahi_core.rs b/drivers/gpu/drm/asahi/asahi_core.rs index a9ed7a204b3b8..3c20741615385 100644 --- a/drivers/gpu/drm/asahi/asahi_core.rs +++ b/drivers/gpu/drm/asahi/asahi_core.rs @@ -37,7 +37,9 @@ mod mmu; mod object; mod pgtable; mod queue; +mod queue_share; mod regs; +mod retain; mod slotalloc; mod stats; mod stats_busy; @@ -97,6 +99,8 @@ impl kernel::InPlaceModule for DriverModule { fn init(module: &'static kernel::ThisModule) -> impl PinInit { // SAFETY: Module init runs once, before either adapter is registered below. unsafe { binding::init() }; + // SAFETY: Module init runs once, before either adapter is registered below. + unsafe { queue_share::init() }; try_pin_init!(Self { // Initialize the queue first, before registration can invoke probe. _cleanup: cleanup::QueueOwner::new()?, @@ -123,6 +127,22 @@ module! { license: "Dual MIT/GPL", alias: ["asahi"], params: { + m5_mesh_storage_chunks: u32 { + default: 0x200, + description: "M5 mesh rendering buffer bound to API task (object-stage) render kicks only, in 64 KiB chunks; 0 disables the binding and the task feature", + }, + m5_thermal_governor: u32 { + default: 0, + description: "M5 thermal flags: bit 0 enables the host GPU power budget controller, bit 2 lets the controller lower the CPU frequency ceiling while the GPU is budget-bound, bit 3 runs the fans at their maximum while it governs; default 0 preserves the existing policy", + }, + m5_thermal_target_mc: u32 { + default: 90000, + description: "M5 GPU power budget governor target for the SoC hotspot temperature computed by the SMC, in milli-degrees C (70000..104000)", + }, + m5_depth_msaa_compression: u32 { + default: 1, + description: "Enable single-layer D32 compressed 2x/4x ZLS admission", + }, debug_flags: u64 { default: 0, // permissions: 0o644, diff --git a/drivers/gpu/drm/asahi/driver.rs b/drivers/gpu/drm/asahi/driver.rs index f07596f9e6b16..2b8bdac35f43d 100644 --- a/drivers/gpu/drm/asahi/driver.rs +++ b/drivers/gpu/drm/asahi/driver.rs @@ -147,6 +147,8 @@ impl drm::driver::Driver for AsahiDrmDriver { pub(crate) enum ProbeConfig { Legacy(&'static hw::HwConfig), G17(&'static g17::Config), + /// M5 Pro, Max and Ultra: the ID registers select the configuration. + G17M5, M3(crate::m3::Chip), /// The M5 (T8142) backend, refused unless it is built. M5, @@ -177,6 +179,14 @@ macro_rules! asahi_of_device_table { of::DeviceId::new(c"apple,agx-t8142"), ProbeConfig::M5 ), + ( + of::DeviceId::new(c"gpu,t6050"), + ProbeConfig::G17M5 + ), + ( + of::DeviceId::new(c"apple,agx-g17s"), + ProbeConfig::G17M5 + ), ( of::DeviceId::new(c"apple,agx-t8103"), ProbeConfig::Legacy(&hw::t8103::HWCONFIG) @@ -273,7 +283,10 @@ impl platform::Driver for AsahiPlatformDriver { let cfg = match info.ok_or(ENODEV)? { ProbeConfig::Legacy(cfg) => *cfg, - ProbeConfig::G17(cfg) => return AsahiPlatformDriver::probe_g17(pdev, cfg), + ProbeConfig::G17(cfg) => { + return AsahiPlatformDriver::probe_g17(pdev, regs::G17Resources::new(pdev)?, cfg) + } + ProbeConfig::G17M5 => return AsahiPlatformDriver::probe_g17_m5(pdev), #[cfg(CONFIG_DRM_ASAHI_M3_T8122)] ProbeConfig::M3(crate::m3::Chip::T8122) => return AsahiPlatformDriver::probe_t8122(pdev), ProbeConfig::M3(chip) => return Err(crate::m3::refuse_probe(pdev, *chip)), @@ -450,14 +463,28 @@ impl AsahiPlatformDriver { }) } + fn probe_g17_m5<'bound>( + pdev: &'bound platform::Device>, + ) -> Result> { + // Every T6050 board binds gpu,t6050. Let the ID registers choose the configuration. + let res = regs::G17Resources::new(pdev)?; + let id = res.get_g17_id()?; + let cfg = hw::t6050::config(&id).ok_or_else(|| { + dev_err!(pdev.as_ref(), "No M5 configuration for GPU variant {:#x} with {} dies\n", + id.variant, id.num_dies); + ENODEV + })?; + Self::probe_g17(pdev, res, cfg) + } + fn probe_g17<'bound>( pdev: &'bound platform::Device>, + res: regs::G17Resources, cfg: &'static g17::Config, ) -> Result> { // SAFETY: The GPU performs DMA through a UAT whose output width is part of the SoC config. unsafe { pdev.dma_set_mask_and_coherent(DmaMask::try_new(cfg.uat_oas)?)? }; let resources = regs::Resources::new(pdev)?; - let res = regs::G17Resources::new(pdev)?; let drm = drm::UnregisteredDevice::::new(pdev, Ok(()))?; // SAFETY: Every adapter calls close_binding in unbind before devres; // probe errors drop this local owner while the bound callback is live. diff --git a/drivers/gpu/drm/asahi/file.rs b/drivers/gpu/drm/asahi/file.rs index cc3ccb374ea11..8c1a0f21ffb25 100644 --- a/drivers/gpu/drm/asahi/file.rs +++ b/drivers/gpu/drm/asahi/file.rs @@ -20,6 +20,7 @@ use crate::{ memacct, mmu, queue, + queue_share, util::{ align, align_down, @@ -30,7 +31,7 @@ use crate::{ }, // }; use core::mem::MaybeUninit; -use core::ops::Deref; +use core::ops::{Deref, DerefMut}; use core::ops::Range; use core::ptr::addr_of_mut; use core::sync::atomic::{AtomicBool, Ordering}; @@ -59,6 +60,77 @@ use kernel::{ xarray, // }; +// The M5 runtime's UAPI numbers. The header's values are Mesa's; the M5 queue and +// GET_PARAMS paths use these names. + +/// DRM_ASAHI_BIND_SPARSE_TILE: with SINGLE_PAGE, fill the range +/// from kernel sparse tiles (read-only: global zero tile; writable: per-VM +/// sink) instead of the GEM's first page. The GEM only anchors the GPUVA. +pub(crate) const DRM_ASAHI_BIND_SPARSE_TILE: u32 = uapi::drm_asahi_bind_flags_DRM_ASAHI_BIND_SPARSE_TILE; +/// DRM_ASAHI_FEATURE_SPARSE_TILES: BIND_SPARSE_TILE accepted. +pub(crate) const DRM_ASAHI_FEATURE_SPARSE_TILES: u64 = + uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_SPARSE_TILES as u64; +pub(crate) const DRM_ASAHI_FEATURE_COMPUTE_MATRIX: u64 = + uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_COMPUTE_MATRIX as u64; +/// DRM_ASAHI_FEATURE_TASK_MESH_STORAGE: render programs accept +/// DRM_ASAHI_RENDER_PROGRAM_TASK and such task (object-stage) kicks get the +/// TA mesh rendering buffer, which backs the object stage's payload address. +/// Reported only while m5_mesh_storage_chunks != 0. +pub(crate) const DRM_ASAHI_FEATURE_TASK_MESH_STORAGE: u64 = + uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_TASK_MESH_STORAGE as u64; +pub(crate) const DRM_ASAHI_FEATURE_RENDER_SHADER_WRITES: u64 = + uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_RENDER_SHADER_WRITES as u64; +/// D32 2x/4x ZLS admission; no compressed stencil or resolve bits. +pub(crate) const DRM_ASAHI_FEATURE_M5_DEPTH_MSAA_COMPRESSION: u64 = + uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_M5_DEPTH_MSAA_COMPRESSION as u64; +/// DRM_ASAHI_FEATURE_M5_STENCIL_MSAA_COMPRESSION: single-layer +/// 2x/4x ZLS admission in the compressed Z/S form, stencil included. +pub(crate) const DRM_ASAHI_FEATURE_M5_STENCIL_MSAA_COMPRESSION: u64 = + uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_M5_STENCIL_MSAA_COMPRESSION as u64; +#[allow(dead_code)] +pub(crate) const DRM_ASAHI_RENDER_SHADER_WRITES: u32 = + uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_SHADER_WRITES; +/// Renders may carry separate tiling prefixes of both subqueues +/// (DRM_ASAHI_RENDER_STAGE_PREFIXES), as the complete struct drm_asahi_cmd_render_m5. +pub(crate) const DRM_ASAHI_FEATURE_RENDER_STAGE_PREFIXES_V2: u64 = + uapi::DRM_ASAHI_FEATURE_RENDER_STAGE_PREFIXES_V2 as u64; +/// Compute commands may declare a stream that ends with a memory-scoped flush +/// (DRM_ASAHI_COMPUTE_VISIBLE_COMPLETION). +pub(crate) const DRM_ASAHI_FEATURE_COMPUTE_VISIBLE_COMPLETION: u64 = + uapi::DRM_ASAHI_FEATURE_COMPUTE_VISIBLE_COMPLETION as u64; +/// The compute command's writes are visible when it completes, so another queue's compute +/// command may wait for it in firmware. +pub(crate) const DRM_ASAHI_COMPUTE_VISIBLE_COMPLETION: u32 = + uapi::drm_asahi_compute_flags_DRM_ASAHI_COMPUTE_VISIBLE_COMPLETION as u32; +/// The tiling half waits only on the render and compute prefixes of the command's stage-prefix +/// fields, its fragment half on the whole ones. +#[allow(dead_code)] +pub(crate) const DRM_ASAHI_RENDER_STAGE_PREFIXES: u32 = + uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_STAGE_PREFIXES; +/// Renders may declare that they write whole attachments only +/// (DRM_ASAHI_RENDER_WHOLE_ATTACHMENTS). +pub(crate) const DRM_ASAHI_FEATURE_WHOLE_ATTACHMENT_RENDERS: u64 = + uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_WHOLE_ATTACHMENT_RENDERS as u64; +/// The render area covers every attachment's full extent, so its tile stores +/// write no pixel outside it: an overlap with another such render without an +/// ordering dependency is the application's own race. +#[allow(dead_code)] +pub(crate) const DRM_ASAHI_RENDER_WHOLE_ATTACHMENTS: u32 = + uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_WHOLE_ATTACHMENTS; +/// DRM_ASAHI_RENDER_PROGRAM_TASK, only with DRM_ASAHI_RENDER_PROGRAM_OBJECT: the object +/// program is an API task stage. +pub(crate) const DRM_ASAHI_RENDER_PROGRAM_TASK: u32 = uapi::DRM_ASAHI_RENDER_PROGRAM_TASK; +/// Bit 16, as Mesa defines it. +#[allow(dead_code)] +pub(crate) const DRM_ASAHI_COMPUTE_USES_MATRIX: u32 = + uapi::drm_asahi_compute_flags_DRM_ASAHI_COMPUTE_USES_MATRIX as u32; +/// The compute command neither copies nor resets queries, so it cannot observe +/// timestamps the host publishes after earlier commands complete +/// (DRM_ASAHI_COMPUTE_NO_QUERY_ACCESS). Accepted, and reported with +/// DRM_ASAHI_FEATURE_COMPUTE_NO_QUERY_ACCESS by the M5 command set. +pub(crate) const DRM_ASAHI_COMPUTE_NO_QUERY_ACCESS: u32 = + uapi::drm_asahi_compute_flags_DRM_ASAHI_COMPUTE_NO_QUERY_ACCESS as u32; + const DEBUG_CLASS: DebugFlags = DebugFlags::File; pub(crate) const MAX_COMMANDS_PER_SUBMISSION: u32 = 64; @@ -264,6 +336,27 @@ pub(crate) enum Object { TimestampBuffer(Arc), } +/// A queue in a file's table. Its count in the creating process's share of the device's +/// queues is given back once the queue itself is gone. +pub(crate) struct TableQueue { + queue: KBox, + _share: Option, +} + +impl Deref for TableQueue { + type Target = dyn queue::Queue; + + fn deref(&self) -> &Self::Target { + &*self.queue + } +} + +impl DerefMut for TableQueue { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut *self.queue + } +} + /// State associated with a client. // #[pin_data] pub(crate) struct File { @@ -273,7 +366,7 @@ pub(crate) struct File { // Fields drop in order: the queues go first, so that a closing client's submitted work // can finish before its VMs unmap the user ranges. // #[pin] - queues: xarray::XArray>>>, + queues: xarray::XArray>>, // #[pin] vms: xarray::XArray>, // #[pin] @@ -364,7 +457,7 @@ impl File { let raw_queues = addr_of_mut!((*slot).queues); pin_init::raw_try_init( raw_queues, - xarray::XArray::>>>::new( + xarray::XArray::>>::new( xarray::AllocKind::Alloc1, ), )?; @@ -393,7 +486,7 @@ impl File { } #[allow(clippy::type_complexity)] - fn queues(self: Pin<&Self>) -> Pin<&xarray::XArray>>>> { + fn queues(self: Pin<&Self>) -> Pin<&xarray::XArray>>> { // SAFETY: Structural pinned projection for queues. // We never move out of this field. unsafe { self.map_unchecked(|s| &s.queues) } @@ -589,7 +682,9 @@ impl File { let gpu = ®_data.gpu; let file_id = file.inner().id; - let vm = gpu.new_vm_for_file(kernel_range.clone(), file_id)?; + let vm = gpu + .new_vm_for_file(kernel_range.clone(), file_id)? + .with_memory_client(file.inner().mem.clone()); // G16 registers a retained backend owner before returning this VM. Any later // allocation can fail before file::Vm exists to run its normal close path. let failed_vm = ScopeGuard::new_with_data( @@ -945,11 +1040,13 @@ impl File { if op.flags & !(uapi::drm_asahi_bind_flags_DRM_ASAHI_BIND_READ | uapi::drm_asahi_bind_flags_DRM_ASAHI_BIND_WRITE - | uapi::drm_asahi_bind_flags_DRM_ASAHI_BIND_SINGLE_PAGE) + | uapi::drm_asahi_bind_flags_DRM_ASAHI_BIND_SINGLE_PAGE + | DRM_ASAHI_BIND_SPARSE_TILE) != 0 { return Err(EINVAL); } + let sparse_tile = op.flags & DRM_ASAHI_BIND_SPARSE_TILE != 0; let read = op.flags & uapi::drm_asahi_bind_flags_DRM_ASAHI_BIND_READ != 0; let write = op.flags & uapi::drm_asahi_bind_flags_DRM_ASAHI_BIND_WRITE != 0; let prot = match (read, write) { @@ -976,6 +1073,7 @@ impl File { op.offset, prot, single_page, + sparse_tile, )?)?; } vm.reserve_deferred_user_unmaps(unmaps)?; @@ -1192,7 +1290,7 @@ impl File { .queues() .find(index, usize::MAX) .map(|(idx, guard)| { - let queue: Arc>> = guard.borrow().into(); + let queue: Arc> = guard.borrow().into(); (idx, queue) }) else { @@ -1420,6 +1518,15 @@ impl File { return Err(EACCES); } + // The queue counts against the creating process's share of the device's independent + // queues, whichever of its files it is created on, so no process can take every + // execution context and leave the others, the compositor included, unable to create + // one (see [`queue_share`]); a refusal fails with EBUSY. + let share = reg_data + .gpu + .queue_limits() + .map(|limits| queue_share::Share::take_current(limits.max_queues)) + .transpose()?; let queues_xa = file.inner().queues(); let resv = queues_xa.lock().reserve_limit(1..=u32::MAX, GFP_KERNEL)?; let vms_xa = file.inner().vms(); @@ -1445,7 +1552,13 @@ impl File { queue.reserve_render_scs(data.render_scs_layout)?; data.queue_id = resv.index().try_into()?; - resv.fill(Arc::pin_init(new_mutex!(queue), GFP_KERNEL)?)?; + resv.fill(Arc::pin_init( + new_mutex!(TableQueue { + queue, + _share: share, + }), + GFP_KERNEL, + )?)?; Ok(0) } @@ -1489,7 +1602,7 @@ impl File { } // Upgrade to Arc to drop the XArray lock early - let queue: Arc>> = file + let queue: Arc> = file .inner() .queues() .lock() diff --git a/drivers/gpu/drm/asahi/fwid.rs b/drivers/gpu/drm/asahi/fwid.rs index 1c4480defadad..d646bff94cf1f 100644 --- a/drivers/gpu/drm/asahi/fwid.rs +++ b/drivers/gpu/drm/asahi/fwid.rs @@ -2,10 +2,15 @@ //! Exact GPU firmware identity. //! -//! A build is the SHA-256 of the loaded TEXT with the 8 entropy bytes of its two entropy records +//! A build is the SHA-256 of the loaded TEXT with the 8 entropy bytes of its entropy records //! (`GKTS` and `ECAP`, each a tag, the length 8 and the bytes) zeroed, plus the TEXT and DATA -//! sizes and the records' offsets. An unknown build is refused with its -//! measured identity, so nothing is written to a firmware the driver was not written for. +//! sizes and the records' offsets. One TEXT region can hold several firmware roles back to back +//! (two GPU roles' TEXT loaded one after the other), each with its own pair of records. A build +//! can also name other ranges the boot loader fills (two sections it patches with the boot +//! arguments and the image header's load fields, which can change from boot to boot); they are +//! zeroed too, so the digest is that of the firmware's TEXT with the same ranges zeroed. An +//! unknown build is refused with its measured identity, so nothing is written to a firmware the +//! driver was not written for. use core::ops::Range; use kernel::{bindings, device, fmt, prelude::*}; @@ -15,9 +20,11 @@ pub(crate) struct FirmwareBuild { pub(crate) name: &'static str, pub(crate) text_size: u64, pub(crate) data_size: u64, - /// Offsets of the GKTS and ECAP entropy records in TEXT; their 8 entropy bytes are zeroed - /// before hashing. - pub(crate) entropy: [u32; 2], + /// Offsets of each role's GKTS and ECAP entropy records in TEXT, in TEXT order; their 8 + /// entropy bytes are zeroed before hashing. + pub(crate) entropy: &'static [[u32; 2]], + /// Other (offset, length) ranges of TEXT the boot loader fills, zeroed before hashing. + pub(crate) boot_filled: &'static [(u32, u32)], pub(crate) text_sha256: [u8; 32], } @@ -65,42 +72,111 @@ pub(crate) fn copy_physical(phys: u64, size: usize) -> Result> { Ok(copy?) } -/// Entropy record tags, in TEXT order. +/// Entropy record tags, in TEXT order within a role. const ENTROPY_TAGS: [[u8; 4]; 2] = [*b"GKTS", *b"ECAP"]; -/// The SHA-256 of `image` with the 8 entropy bytes of the records at `at` zeroed, or `None` if a -/// record (tag, then the length 8) is not there. `image` is left as it was. -fn digest_at(image: &mut [u8], at: [u32; 2]) -> Option<[u8; 32]> { - let mut saved = [[0u8; 8]; 2]; - for ((off, tag), save) in at.into_iter().zip(ENTROPY_TAGS).zip(saved.iter_mut()) { - let off = off as usize; - if image.get(off..off + 4) != Some(tag.as_slice()) - || image.get(off + 4..off + 8) != Some(8u32.to_le_bytes().as_slice()) - { +/// Firmware roles one TEXT region can hold. +const MAX_ROLES: usize = 2; + +/// The record (tag, then the length 8) that holds a role's entropy of `tag`. +fn record(tag: [u8; 4]) -> [u8; 8] { + let mut record = [0u8; 8]; + record[..4].copy_from_slice(&tag); + record[4..].copy_from_slice(&8u32.to_le_bytes()); + record +} + +/// Whether `image` holds a whole entropy record of `tag` at `off`: the tag, the length 8 and the +/// 8 entropy bytes, all inside `image`. +fn record_at(image: &[u8], off: usize, tag: [u8; 4]) -> bool { + off.checked_add(16) + .and_then(|end| image.get(off..end)) + .is_some_and(|whole| whole[..8] == record(tag)) +} + +/// The SHA-256 of `image` with the 8 entropy bytes of the records at `at` (one GKTS/ECAP pair +/// per role) and the `filled` ranges zeroed, or `None` if a record or range is not there. +/// `image` is left as it was. +fn digest_at(image: &mut [u8], at: &[[u32; 2]], filled: &[(u32, u32)]) -> Option<[u8; 32]> { + if at.is_empty() || at.len() > MAX_ROLES { + return None; + } + if !filled.is_empty() { + for (off, tag) in at.iter().flat_map(|pair| pair.iter().zip(ENTROPY_TAGS)) { + if !record_at(image, *off as usize, tag) { + return None; + } + } + let mut copy = KVVec::from_elem(0u8, image.len(), GFP_KERNEL).ok()?; + copy.copy_from_slice(image); + for off in at.iter().flatten() { + copy[*off as usize + 8..*off as usize + 16].fill(0); + } + for &(off, len) in filled { + let off = off as usize; + copy.get_mut(off..off.checked_add(len as usize)?)?.fill(0); + } + return Some(sha256(©)); + } + let records = || { + at.iter() + .flat_map(|pair| pair.iter().map(|&off| off as usize).zip(ENTROPY_TAGS)) + }; + let mut saved = [[0u8; 8]; 2 * MAX_ROLES]; + for ((off, tag), save) in records().zip(saved.iter_mut()) { + if !record_at(image, off, tag) { return None; } - save.copy_from_slice(image.get(off + 8..off + 16)?); + save.copy_from_slice(&image[off + 8..off + 16]); } - for off in at { - image[off as usize + 8..off as usize + 16].fill(0); + for (off, _) in records() { + image[off + 8..off + 16].fill(0); } let digest = sha256(image); - for (off, save) in at.into_iter().zip(saved) { - image[off as usize + 8..off as usize + 16].copy_from_slice(&save); + for ((off, _), save) in records().zip(saved) { + image[off + 8..off + 16].copy_from_slice(&save); } Some(digest) } -/// The first record of each tag in `image`, to report an unknown build. -fn find_entropy(image: &[u8]) -> Option<[u32; 2]> { - let mut found = [0u32; 2]; - for (slot, tag) in found.iter_mut().zip(ENTROPY_TAGS) { - let mut record = [0u8; 8]; - record[..4].copy_from_slice(&tag); - record[4..].copy_from_slice(&8u32.to_le_bytes()); - *slot = u32::try_from(image.windows(8).position(|w| w == record)?).ok()?; +/// The entropy records of each role found in `image`, to report an unknown build: each GKTS +/// record with the first ECAP record after it, at most [`MAX_ROLES`] pairs. +struct Records { + at: [[u32; 2]; MAX_ROLES], + roles: usize, +} + +impl Records { + fn find(image: &[u8]) -> Option { + let [gkts, ecap] = ENTROPY_TAGS.map(record); + let mut found = Records { + at: [[0; 2]; MAX_ROLES], + roles: 0, + }; + let mut from = 0; + while found.roles < MAX_ROLES { + let Some(g) = image.get(from..)?.windows(8).position(|w| w == gkts) else { + break; + }; + let g = from + g; + let Some(e) = image.get(g + 8..)?.windows(8).position(|w| w == ecap) else { + break; + }; + let e = g + 8 + e; + // A record whose entropy bytes run past TEXT is no record ([`record_at`]). + if !record_at(image, g, ENTROPY_TAGS[0]) || !record_at(image, e, ENTROPY_TAGS[1]) { + break; + } + found.at[found.roles] = [u32::try_from(g).ok()?, u32::try_from(e).ok()?]; + found.roles += 1; + from = e + 8; + } + (found.roles > 0).then_some(found) + } + + fn as_slice(&self) -> &[[u32; 2]] { + &self.at[..self.roles] } - Some(found) } /// Finds an exact identity, measuring each candidate at its own entropy offsets. @@ -108,54 +184,128 @@ fn lookup( builds: &'static [FirmwareBuild], text_size: u64, data_size: u64, - mut digest: impl FnMut([u32; 2]) -> Option<[u8; 32]>, + mut digest: impl FnMut(&FirmwareBuild) -> Option<[u8; 32]>, ) -> Option<&'static FirmwareBuild> { builds.iter().find(|build| { build.text_size == text_size && build.data_size == data_size - && digest(build.entropy) == Some(build.text_sha256) + && digest(build) == Some(build.text_sha256) + }) +} + +/// A private, stable copy of a loaded TEXT, with the size of its DATA. +pub(crate) struct Measured { + text: KVVec, + data_size: u64, +} + +/// Copies the TEXT loaded at `text` (DATA `data_size` bytes) twice, before any firmware-visible +/// write, and keeps the copy if both agree (`EAGAIN` if the TEXT changed meanwhile). +pub(crate) fn measure(text: Range, data_size: u64) -> Result { + let size = usize::try_from(text.end.checked_sub(text.start).ok_or(EINVAL)?)?; + let copy = copy_physical(text.start, size)?; + let again = copy_physical(text.start, size)?; + if copy.as_slice() != again.as_slice() { + return Err(EAGAIN); + } + Ok(Measured { + text: copy, + data_size, }) } +impl Measured { + /// The build among `builds` this TEXT is exactly, if any. + pub(crate) fn find( + &mut self, + builds: &'static [FirmwareBuild], + ) -> Option<&'static FirmwareBuild> { + let text_size = self.text.len() as u64; + let text = &mut self.text; + lookup(builds, text_size, self.data_size, |build| { + digest_at(&mut text[..], build.entropy, build.boot_filled) + }) + } + + /// The measured identity, to report a build that is not admitted: normalized as the first + /// of `builds` with this TEXT size would be, else with only its entropy records zeroed. + pub(crate) fn identity( + &mut self, + builds: impl IntoIterator, + ) -> Identity { + let text_size = self.text.len() as u64; + let filled = builds + .into_iter() + .find(|build| build.text_size == text_size && !build.boot_filled.is_empty()) + .map_or(&[][..], |build| build.boot_filled); + let records = Records::find(&self.text); + let digest = records + .as_ref() + .and_then(|r| digest_at(&mut self.text, r.as_slice(), filled)); + Identity { + text_size, + data_size: self.data_size, + filled: filled.len(), + measured: records.zip(digest), + } + } +} + +/// A measured identity, as a pinned [`FirmwareBuild`] would state it. +pub(crate) struct Identity { + text_size: u64, + data_size: u64, + /// How many boot-filled ranges were zeroed besides the entropy. + filled: usize, + measured: Option<(Records, [u8; 32])>, +} + +impl fmt::Display for Identity { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "TEXT {:#x} DATA {:#x}", self.text_size, self.data_size)?; + let Some((records, digest)) = &self.measured else { + return write!(f, " has no entropy records"); + }; + write!(f, " entropy")?; + for (i, [gkts, ecap]) in records.as_slice().iter().enumerate() { + write!( + f, + "{}{:#x}/{:#x}", + if i == 0 { " " } else { "," }, + gkts, + ecap + )?; + } + if self.filled != 0 { + write!(f, " ({} boot-filled ranges zeroed)", self.filled)?; + } + write!(f, " sha256 ")?; + fmt::Display::fmt(&Hex(digest), f) + } +} + /// Identifies the firmware build loaded at `text` (DATA `data_size` bytes) among `builds`, /// before any firmware-visible write. +#[cfg_attr(not(CONFIG_DRM_ASAHI_G16), allow(dead_code))] pub(crate) fn identify( dev: &device::Device, text: Range, data_size: u64, builds: &'static [FirmwareBuild], ) -> Result<&'static FirmwareBuild> { - let size = usize::try_from(text.end - text.start)?; - let mut copy = copy_physical(text.start, size)?; - let again = copy_physical(text.start, size)?; - if copy.as_slice() != again.as_slice() { - dev_err!(dev, "GPU firmware TEXT changed while it was measured\n"); - return Err(EAGAIN); - } - if let Some(build) = lookup(builds, size as u64, data_size, |at| { - digest_at(&mut copy, at) - }) { + let mut measured = measure(text, data_size).inspect_err(|e| { + if *e == EAGAIN { + dev_err!(dev, "GPU firmware TEXT changed while it was measured\n"); + } + })?; + if let Some(build) = measured.find(builds) { dev_info!(dev, "GPU firmware build: {}\n", build.name); return Ok(build); } - let Some((entropy, digest)) = - find_entropy(©).and_then(|at| Some((at, digest_at(&mut copy, at)?))) - else { - dev_err!( - dev, - "unsupported GPU firmware build: TEXT {:#x} has no entropy records\n", - size - ); - return Err(EINVAL); - }; dev_err!( dev, - "unsupported GPU firmware build: TEXT {:#x} DATA {:#x} entropy {:#x}/{:#x} sha256 {}\n", - size, - data_size, - entropy[0], - entropy[1], - Hex(&digest) + "unsupported GPU firmware build: {}\n", + measured.identity(builds) ); for build in builds { dev_err!(dev, "supported: {}\n", build.name); @@ -167,48 +317,120 @@ pub(crate) fn identify( mod tests { use super::*; - fn image() -> [u8; 64] { - let mut image = [0x5a; 64]; - for (off, tag) in [0usize, 32].into_iter().zip(ENTROPY_TAGS) { - image[off..off + 4].copy_from_slice(&tag); - image[off + 4..off + 8].copy_from_slice(&8u32.to_le_bytes()); + /// `roles` GKTS/ECAP pairs, at 0/32 within each 64 bytes. + fn image() -> [u8; N] { + let mut image = [0x5a; N]; + for role in 0..N / 64 { + for (off, tag) in [0usize, 32].into_iter().zip(ENTROPY_TAGS) { + image[role * 64 + off..role * 64 + off + 8].copy_from_slice(&record(tag)); + } } image } #[test] fn entropy_is_normalized_without_changing_the_image() { - let mut first = image(); + let mut first = image::<64>(); let original = first; - let digest = digest_at(&mut first, [0, 32]).unwrap(); + let digest = digest_at(&mut first, &[[0, 32]], &[]).unwrap(); assert_eq!(first, original); let mut second = first; second[8..16].fill(0xff); second[40..48].fill(0x01); - assert_eq!(digest_at(&mut second, [0, 32]), Some(digest)); + assert_eq!(digest_at(&mut second, &[[0, 32]], &[]), Some(digest)); second[63] ^= 1; - assert_ne!(digest_at(&mut second, [0, 32]), Some(digest)); + assert_ne!(digest_at(&mut second, &[[0, 32]], &[]), Some(digest)); + } + + #[test] + fn every_role_is_normalized() { + let mut first = image::<128>(); + let original = first; + let both = &[[0, 32], [64, 96]]; + let digest = digest_at(&mut first, both, &[]).unwrap(); + assert_eq!(first, original); + let mut second = first; + second[72..80].fill(0xff); + second[104..112].fill(0x01); + assert_eq!(digest_at(&mut second, both, &[]), Some(digest)); + // The second role's entropy is part of the digest unless its records are named. + assert_ne!( + digest_at(&mut second, &[[0, 32]], &[]), + digest_at(&mut first, &[[0, 32]], &[]) + ); + second[127] ^= 1; + assert_ne!(digest_at(&mut second, both, &[]), Some(digest)); + let found = Records::find(&first).unwrap(); + assert_eq!(found.as_slice(), both); + assert_eq!( + digest_at(&mut first, &[[0, 32], [64, 96], [0, 32]], &[]), + None + ); + } + + #[test] + fn boot_filled_ranges_are_normalized() { + let mut first = image::<128>(); + let original = first; + let filled = &[(16, 8), (100, 28)]; + let digest = digest_at(&mut first, &[[0, 32]], filled).unwrap(); + assert_eq!(first, original); + let mut second = first; + second[16..24].fill(0x11); + second[100..128].fill(0x22); + assert_eq!(digest_at(&mut second, &[[0, 32]], filled), Some(digest)); + second[99] ^= 1; + assert_ne!(digest_at(&mut second, &[[0, 32]], filled), Some(digest)); + assert_eq!(digest_at(&mut first, &[[0, 32]], &[(120, 9)]), None); + } + + #[test] + fn a_record_without_its_entropy_bytes_is_refused() { + // The role's ECAP record is in the last 8 bytes, so it has no entropy bytes. + let mut tail = image::<64>(); + tail[32..40].fill(0x5a); + tail[56..64].copy_from_slice(&record(ENTROPY_TAGS[1])); + let original = tail; + assert_eq!(digest_at(&mut tail, &[[0, 56]], &[(16, 8)]), None); + assert_eq!(digest_at(&mut tail, &[[0, 56]], &[]), None); + assert_eq!(tail, original); + assert!(Records::find(&tail).is_none()); + // An unknown build with a pinned build's TEXT size is reported, not a panic. + static PINNED: [FirmwareBuild; 1] = [FirmwareBuild { + name: "pinned", + text_size: 64, + data_size: 0, + entropy: &[[0, 32]], + boot_filled: &[(16, 8)], + text_sha256: [0; 32], + }]; + let mut text = KVVec::from_elem(0u8, tail.len(), GFP_KERNEL).unwrap(); + text.copy_from_slice(&tail); + let mut measured = Measured { text, data_size: 0 }; + assert!(measured.find(&PINNED).is_none()); + assert!(measured.identity(&PINNED).measured.is_none()); } #[test] fn malformed_records_are_refused_without_changing_the_image() { - let mut image = image(); - assert_eq!(digest_at(&mut image, [0, 56]), None); + let mut image = image::<64>(); + assert_eq!(digest_at(&mut image, &[[0, 56]], &[]), None); + assert_eq!(digest_at(&mut image, &[], &[]), None); image[36] = 7; let original = image; - assert_eq!(digest_at(&mut image, [0, 32]), None); + assert_eq!(digest_at(&mut image, &[[0, 32]], &[]), None); assert_eq!(image, original); - assert_eq!(find_entropy(&image), None); + assert!(Records::find(&image).is_none()); } fn measured( builds: &'static [FirmwareBuild], data_size: u64, - entropy: [u32; 2], + entropy: &[[u32; 2]], digest: [u8; 32], ) -> Option<&'static FirmwareBuild> { - lookup(builds, 0x60000, data_size, |at| { - (at == entropy).then_some(digest) + lookup(builds, 0x60000, data_size, |build| { + (build.entropy == entropy).then_some(digest) }) } @@ -238,13 +460,13 @@ mod tests { ], ]; for (i, digest) in digests.into_iter().enumerate() { - let found = measured(build.images, 0x138000, [0x59839, 0x5991c], digest).unwrap(); + let found = measured(build.images, 0x138000, &[[0x59839, 0x5991c]], digest).unwrap(); assert!(core::ptr::eq(found, &build.images[i])); - assert!(measured(build.images, 0x13c000, [0x59839, 0x5991c], digest).is_none()); - assert!(measured(build.images, 0x138000, [0x5c8df, 0x5c9c2], digest).is_none()); + assert!(measured(build.images, 0x13c000, &[[0x59839, 0x5991c]], digest).is_none()); + assert!(measured(build.images, 0x138000, &[[0x5c8df, 0x5c9c2]], digest).is_none()); assert!(lookup(build.images, 0x60001, 0x138000, |_| Some(digest)).is_none()); } - assert!(measured(build.images, 0x138000, [0x59839, 0x5991c], [0; 32]).is_none()); + assert!(measured(build.images, 0x138000, &[[0x59839, 0x5991c]], [0; 32]).is_none()); } #[test] @@ -311,10 +533,56 @@ mod tests { ], ]; for (i, digest) in digests.into_iter().enumerate() { - let found = measured(builds, 0x13c000, [0x5c8df, 0x5c9c2], digest).unwrap(); + let found = measured(builds, 0x13c000, &[[0x5c8df, 0x5c9c2]], digest).unwrap(); assert!(core::ptr::eq(found, &builds[i])); - assert!(measured(builds, 0x138000, [0x5c8df, 0x5c9c2], digest).is_none()); + assert!(measured(builds, 0x138000, &[[0x5c8df, 0x5c9c2]], digest).is_none()); + } + assert!(measured(builds, 0x13c000, &[[0x5c8df, 0x5c9c2]], [0; 32]).is_none()); + } + + #[test] + fn t6050_admits_only_its_pinned_image() { + for cfg in [ + &crate::hw::t6050::CONFIG_ULTRA, + &crate::hw::t6050::CONFIG_MAX, + &crate::hw::t6050::CONFIG, + ] { + let crate::g17::fw::variant::Firmware::Identified(builds) = cfg.firmware else { + panic!("an M5 board on bare metal must identify its loaded firmware"); + }; + t6050_admits_only(builds); + } + } + + fn t6050_admits_only(builds: &[crate::g17::fw::variant::Build]) { + for image in builds.iter().flat_map(|build| build.images) { + // Both roles' records, inside TEXT. + assert_eq!(image.entropy.len(), 2); + assert!(image + .entropy + .iter() + .flatten() + .all(|&o| u64::from(o) + 16 <= image.text_size)); + let admit = |text, data, at: &[[u32; 2]], digest| { + lookup(core::slice::from_ref(image), text, data, |m| { + (m.entropy == at).then_some(digest) + }) + }; + // The header fields and both boot-filled sections of each role. + assert_eq!(image.boot_filled.len(), 8); + let (t, d, e, h) = ( + image.text_size, + image.data_size, + image.entropy, + image.text_sha256, + ); + assert!(core::ptr::eq(admit(t, d, e, h).unwrap(), image)); + let mut wrong = h; + wrong[31] ^= 1; + assert!(admit(t, d, e, wrong).is_none()); + assert!(admit(t, d + 0x4000, e, h).is_none()); + assert!(admit(t + 0x4000, d, e, h).is_none()); + assert!(admit(t, d, &e[..1], h).is_none()); } - assert!(measured(builds, 0x13c000, [0x5c8df, 0x5c9c2], [0; 32]).is_none()); } } diff --git a/drivers/gpu/drm/asahi/g15/boot.rs b/drivers/gpu/drm/asahi/g15/boot.rs index 70decd06fca0d..95269e152177b 100644 --- a/drivers/gpu/drm/asahi/g15/boot.rs +++ b/drivers/gpu/drm/asahi/g15/boot.rs @@ -61,8 +61,6 @@ pub(crate) mod glue { fn asahi_g15_machine(compatible: *const core::ffi::c_char) -> bool; fn asahi_g15_hold_power(dev: *mut core::ffi::c_void); fn asahi_g15_release_power(dev: *mut core::ffi::c_void); - fn asahi_g15_module_try_get() -> bool; - fn asahi_g15_module_put(); pub(crate) fn asahi_g15_reprobe(); pub(crate) fn asahi_g15_report_loader_gpu(); fn asahi_g15_adt_phram(start: *mut u64, size: *mut u64) -> core::ffi::c_int; @@ -127,14 +125,7 @@ pub(crate) mod glue { /// Pins this module until reboot, once (`try_module_get(THIS_MODULE)`, see `retain.rs`). /// ENODEV, with nothing pinned or marked, once the module's removal has begun. pub(crate) fn retain_module() -> Result { - crate::g15::retain::retain( - &RETAINED, - // SAFETY: Takes a reference to this module unless its removal has begun. - || unsafe { asahi_g15_module_try_get() }, - // SAFETY: Drops only the reference this call took; the winner's is never put. - || unsafe { asahi_g15_module_put() }, - ) - .map_err(|_| ENODEV) + crate::retain::retain_module(&RETAINED) } /// The phram "adt" export: start and size. diff --git a/drivers/gpu/drm/asahi/g15/glue.c b/drivers/gpu/drm/asahi/g15/glue.c index 095b9cb96d51f..2a0346228c1e5 100644 --- a/drivers/gpu/drm/asahi/g15/glue.c +++ b/drivers/gpu/drm/asahi/g15/glue.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT /* - * G15 (T6030/T6031, M3 Pro/Max) helpers that have no Rust bindings: static inlines, - * macros and THIS_MODULE. Built only with CONFIG_DRM_ASAHI_G15. + * G15 (T6030/T6031, M3 Pro/Max) helpers that have no Rust bindings: static inlines + * and macros. The module pin is retain.c's. Built only with CONFIG_DRM_ASAHI_G15. */ #include #include @@ -26,8 +26,6 @@ bool asahi_g15_board(void); bool asahi_g15_machine(const char *compatible); void asahi_g15_hold_power(struct device *dev); void asahi_g15_release_power(struct device *dev); -bool asahi_g15_module_try_get(void); -void asahi_g15_module_put(void); void asahi_g15_reprobe(void); void asahi_g15_report_loader_gpu(void); int asahi_g15_adt_phram(u64 *start, u64 *size); @@ -247,22 +245,6 @@ void asahi_g15_release_power(struct device *dev) } -/* - * Pin this module until reboot, unless its removal has begun: a deferred or - * asynchronous probe can run after an ordinary rmmod has passed its reference - * check, which __module_get() would not stop. - */ -bool asahi_g15_module_try_get(void) -{ - return try_module_get(THIS_MODULE); -} - -/* Drop a reference asahi_g15_module_try_get() took. */ -void asahi_g15_module_put(void) -{ - module_put(THIS_MODULE); -} - /* * The boot loader's phram "adt" export: the first /reserved-memory child * compatible with "phram" and labelled "adt". diff --git a/drivers/gpu/drm/asahi/g15/mod.rs b/drivers/gpu/drm/asahi/g15/mod.rs index ae0512de0b156..861668429197d 100644 --- a/drivers/gpu/drm/asahi/g15/mod.rs +++ b/drivers/gpu/drm/asahi/g15/mod.rs @@ -45,7 +45,6 @@ mod adt; mod board; mod boot; -mod retain; // One admission rule for the generation mappings and shared adopted UAT. pub(crate) use boot::glue::memory_mode; diff --git a/drivers/gpu/drm/asahi/g16/boot.rs b/drivers/gpu/drm/asahi/g16/boot.rs index 0e4d315f64f36..509ed3368df77 100644 --- a/drivers/gpu/drm/asahi/g16/boot.rs +++ b/drivers/gpu/drm/asahi/g16/boot.rs @@ -44,8 +44,6 @@ pub(crate) mod glue { fn asahi_g16_board() -> bool; fn asahi_g16_display_prepare(dev: *mut core::ffi::c_void) -> core::ffi::c_int; fn asahi_g16_hold_power(dev: *mut core::ffi::c_void) -> core::ffi::c_int; - fn asahi_g16_module_try_get() -> bool; - fn asahi_g16_module_put(); fn asahi_g16_ram_disjoint(start: u64, size: u64) -> bool; fn asahi_g16_ioremap_wc(start: u64, size: usize) -> *mut core::ffi::c_void; fn asahi_g16_dma_unsupported(dev: *mut core::ffi::c_void) -> bool; @@ -87,21 +85,10 @@ pub(crate) mod glue { RETAINED.load(Ordering::Acquire) } - /// Pins this module until reboot, once (`try_module_get(THIS_MODULE)`). + /// Pins this module until reboot, once (`try_module_get(THIS_MODULE)`, see `retain.rs`). /// ENODEV, with nothing pinned or marked, once the module's removal has begun. pub(crate) fn retain_module() -> Result { - if RETAINED.load(Ordering::Acquire) { - return Ok(()); - } - // SAFETY: Takes a reference to this module unless its removal has begun. - if !unsafe { asahi_g16_module_try_get() } { - return Err(ENODEV); - } - if RETAINED.swap(true, Ordering::AcqRel) { - // SAFETY: Drops only the reference this call took; the first one is never put. - unsafe { asahi_g16_module_put() }; - } - Ok(()) + crate::retain::retain_module(&RETAINED) } /// The physical range is disjoint from System RAM. @@ -495,7 +482,8 @@ pub(crate) fn admit( } let regs = regs::Regs::new(pdev, res)?; // Once power admission starts, even a rejected cold check must not detach the GFX - // domain under an inherited firmware owner. Keep the module and platform binding. + // domain under an inherited firmware owner. Keep the module and platform binding; once + // the module's removal has begun nothing is held. glue::retain_module()?; glue::hold_power(dev)?; let (power_offset, power_state) = glue::power_domain_state(dev)?; @@ -741,9 +729,12 @@ pub(crate) fn start( return Err(e); } - // From here on the firmware is touched: nothing it can reach is freed. - // Admission already pinned the module, so this cannot fail. - let _ = glue::retain_module(); + // From here on the firmware is touched: nothing it can reach is freed. Admission pinned + // the module, so this only confirms it. + if let Err(e) = glue::retain_module() { + core::mem::forget(objects); + return Err(e); + } let rtkit = match Rtkit::new(&d, shared) { Ok(rtkit) => rtkit, Err(e) => { diff --git a/drivers/gpu/drm/asahi/g16/glue.c b/drivers/gpu/drm/asahi/g16/glue.c index 82eb76a196da8..ad258092c035b 100644 --- a/drivers/gpu/drm/asahi/g16/glue.c +++ b/drivers/gpu/drm/asahi/g16/glue.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT /* - * G16 (T8132, M4) helpers that have no Rust bindings: static inlines, - * macros and THIS_MODULE. Built only with CONFIG_DRM_ASAHI_G16. + * G16 (T8132, M4) helpers that have no Rust bindings: static inlines + * and macros. The module pin is retain.c's. Built only with CONFIG_DRM_ASAHI_G16. */ #include #include @@ -27,8 +27,6 @@ bool asahi_g16_board(void); int asahi_g16_display_prepare(struct device *dev); int asahi_g16_hold_power(struct device *dev); -bool asahi_g16_module_try_get(void); -void asahi_g16_module_put(void); bool asahi_g16_ram_disjoint(u64 start, u64 size); void __iomem *asahi_g16_ioremap_wc(u64 start, size_t size); bool asahi_g16_dma_unsupported(struct device *dev); @@ -197,21 +195,6 @@ int asahi_g16_hold_power(struct device *dev) return dev->pm_domain ? 0 : -EINVAL; } -/* - * Pin this module until reboot, unless its removal has begun: a deferred - * probe can run after rmmod has passed its reference check. - */ -bool asahi_g16_module_try_get(void) -{ - return try_module_get(THIS_MODULE); -} - -/* Drop a reference asahi_g16_module_try_get() took. */ -void asahi_g16_module_put(void) -{ - module_put(THIS_MODULE); -} - /* The range is disjoint from System RAM (ADT region admission). */ bool asahi_g16_ram_disjoint(u64 start, u64 size) { diff --git a/drivers/gpu/drm/asahi/g16/hw.rs b/drivers/gpu/drm/asahi/g16/hw.rs index 696b4bc9f4062..f32d18874e908 100644 --- a/drivers/gpu/drm/asahi/g16/hw.rs +++ b/drivers/gpu/drm/asahi/g16/hw.rs @@ -42,7 +42,8 @@ pub(crate) mod t8132 { name: "Apple GFX 25G83 (normalized loaded image)", text_size: 0x60000, data_size: 0x13c000, - entropy: [0x5c8df, 0x5c9c2], + entropy: &[[0x5c8df, 0x5c9c2]], + boot_filled: &[], text_sha256: [ 0x5f, 0x65, 0xf6, 0x14, 0x58, 0x72, 0x54, 0x70, 0xb4, 0xb0, 0xc0, 0x20, 0x28, 0x47, 0x7b, 0xf2, 0xea, 0x57, 0x3a, 0xfb, 0x4e, 0x0c, 0x49, 0x6e, 0xf9, 0x95, @@ -55,7 +56,8 @@ pub(crate) mod t8132 { name: "Apple GFX 25G83 (normalized loaded image, 8-core J713)", text_size: 0x60000, data_size: 0x13c000, - entropy: [0x5c8df, 0x5c9c2], + entropy: &[[0x5c8df, 0x5c9c2]], + boot_filled: &[], text_sha256: [ 0x3b, 0x36, 0x8e, 0xe4, 0x2d, 0xcc, 0xc6, 0x26, 0xe8, 0x49, 0x67, 0xff, 0x40, 0x87, 0x64, 0x71, 0x0a, 0x64, 0xbf, 0xf1, 0x59, 0x4f, 0x3c, 0xfd, 0x2c, 0x6e, diff --git a/drivers/gpu/drm/asahi/g16/mod.rs b/drivers/gpu/drm/asahi/g16/mod.rs index 4cf9057f04fb0..38fd612d240da 100644 --- a/drivers/gpu/drm/asahi/g16/mod.rs +++ b/drivers/gpu/drm/asahi/g16/mod.rs @@ -173,7 +173,8 @@ impl platform::Driver for Driver { fn unbind<'bound>(_pdev: &'bound platform::Device>, this: Pin<&Self::Data<'bound>>) { if let Some(driver) = this.get_ref() { if let Some(gpu) = &driver.g16 { - gpu.retain(); + // Admission pinned the module; this only confirms the pin. + let _ = gpu.retain(); } driver.close_binding(); } @@ -415,18 +416,17 @@ impl Gpu { }, GFP_KERNEL, )?; - gpu.retain(); + gpu.retain()?; Ok(gpu) } /// Keeps the firmware session and this module alive until reboot. /// - /// Takes the module reference once. Nothing firmware-reachable is freed on any path: the - /// RTKit callbacks hold [`Shared`], which holds the [`Firmware`] (or the [`boot::Orphan`] of a - /// failed start) until reboot. - pub(crate) fn retain(&self) { - // Admission already pinned the module, so this cannot fail. - let _ = boot::glue::retain_module(); + /// Takes the module reference once; admission already took it, so this only confirms it. + /// Nothing firmware-reachable is freed on any path: the RTKit callbacks hold [`Shared`], + /// which holds the [`Firmware`] (or the [`boot::Orphan`] of a failed start) until reboot. + pub(crate) fn retain(&self) -> Result { + boot::glue::retain_module() } } diff --git a/drivers/gpu/drm/asahi/g17/availability.rs b/drivers/gpu/drm/asahi/g17/availability.rs index c76a20db7007a..9deaac406a5a2 100644 --- a/drivers/gpu/drm/asahi/g17/availability.rs +++ b/drivers/gpu/drm/asahi/g17/availability.rs @@ -98,14 +98,33 @@ impl Availability { /// Wakes every waiter of this domain. Call after dropping the resource's own lock; fence /// callbacks run outside the domain lock. pub(crate) fn notify(&self) { - let mut waiters = { + self.advance(); + } + + /// [`Self::notify`] for a caller that sampled `observed` and is giving back its own claim, + /// never published, before it waits. Returns the sample it waits on next: the new + /// generation if nothing else changed the domain since `observed`, so its own notification + /// does not end its wait at once, and `observed` otherwise, so it retries. + pub(crate) fn notify_own(&self, observed: u64) -> u64 { + if self.advance() == observed { + observed.wrapping_add(1) + } else { + observed + } + } + + /// Notifies, returning the generation before this notification. + fn advance(&self) -> u64 { + let (previous, mut waiters) = { let mut state = self.state.lock(); - state.generation = state.generation.wrapping_add(1); - state.waiters.take() + let previous = state.generation; + state.generation = previous.wrapping_add(1); + (previous, state.waiters.take()) }; while let Some(mut entry) = waiters { waiters = entry.next.take(); entry.fence.signal(); } + previous } } diff --git a/drivers/gpu/drm/asahi/g17/buffer.rs b/drivers/gpu/drm/asahi/g17/buffer.rs index fd6460db2d1fb..0747a1fe69c87 100644 --- a/drivers/gpu/drm/asahi/g17/buffer.rs +++ b/drivers/gpu/drm/asahi/g17/buffer.rs @@ -9,6 +9,9 @@ use super::fw::buffer as fw; use crate::hw::t8140::resources as cfg; pub(crate) mod target; +mod geometry; +pub(crate) use geometry::BlockGeometry; +mod layout; use kernel::prelude::*; @@ -19,6 +22,8 @@ struct BufferId { references: u8, owner: bool, token: u64, + /// Hardware context and PB state named by this token's M5 binding. + bound: Option<(u32, u64)>, } impl BufferId { fn free(self) -> bool { @@ -40,6 +45,7 @@ impl BufferIds { references: 0, owner: false, token: 0, + bound: None, }), returned <- pin_init::init_array_from_fn(|index| (BUFFER_IDS - index - 1) as u8), returned_count: BUFFER_IDS, @@ -88,6 +94,7 @@ impl BufferIds { references: 1, owner: false, token, + bound: None, }; Ok((id, token)) } @@ -101,6 +108,7 @@ impl BufferIds { if entry.free() { let token = self.fresh_token(); self.entries[usize::from(id)].token = token; + self.entries[usize::from(id)].bound = None; } self.entries[usize::from(id)].owner = true; self.remove_returned(id); @@ -114,6 +122,28 @@ impl BufferIds { entry.references = entry.references.checked_add(1).ok_or(ENOSPC)?; Ok((id, entry.token)) } + /// A repeated token must name the same M5 PB state and hardware context. + /// Renew it if an idle VM moved; other firmware profiles keep acquire_owner. + pub(crate) fn acquire_owner_bound(&mut self, id: u8, context: u32, pb_va: u64) + -> Result<(u8, u64)> { + let entry = self.entries.get(usize::from(id)).ok_or(EINVAL)?; + if !entry.owner { + return Err(EINVAL); + } + let references = entry.references.checked_add(1).ok_or(ENOSPC)?; + let previous = entry.bound; + let old_token = entry.token; + let token = if previous.is_some_and(|bound| bound != (context, pb_va)) { + self.fresh_token() + } else { + old_token + }; + let entry = &mut self.entries[usize::from(id)]; + entry.references = references; + entry.token = token; + entry.bound = Some((context, pb_va)); + Ok((id, token)) + } /// Preflight shared completion bookkeeping before any of its counters change. pub(crate) fn can_release(&self, id: u8) -> Result { let entry = self.entries.get(usize::from(id)).ok_or(EINVAL)?; @@ -146,16 +176,22 @@ impl BufferIds { /// Each scene and its shorter scratch slot are one lifetime. A completed /// paired render releases both; merely publishing a later scene does not. pub(crate) struct Scenes { - owners: [Option<(u64, u8)>; fw::SCRATCH_SLOTS], + owners: [Option<(u64, u8)>; fw::SCENES], + scratch_slots: usize, next: usize, } impl Scenes { pub(crate) const fn new() -> Self { Self { - owners: [None; fw::SCRATCH_SLOTS], + owners: [None; fw::SCENES], + scratch_slots: fw::SCRATCH_SLOTS, next: fw::FIRST_SCENE, } } + fn with_slots(scratch_slots: usize) -> Result { + if scratch_slots == 0 || scratch_slots > fw::SCENES { return Err(EINVAL); } + Ok(Self { scratch_slots, ..Self::new() }) + } pub(crate) fn select(&self, ordinal: u64) -> Result { if let Some((_, scene)) = self .owners @@ -167,14 +203,14 @@ impl Scenes { } (0..fw::SCENES) .map(|offset| (self.next + offset) % fw::SCENES) - .find(|scene| self.owners[scene % fw::SCRATCH_SLOTS].is_none()) + .find(|scene| self.owners[scene % self.scratch_slots].is_none()) .ok_or(ENOSPC) } pub(crate) fn commit(&mut self, ordinal: u64, scene: usize) -> Result { if self.select(ordinal)? != scene { return Err(EINVAL); } - let slot = &mut self.owners[scene % fw::SCRATCH_SLOTS]; + let slot = &mut self.owners[scene % self.scratch_slots]; if slot.is_none() { *slot = Some((ordinal, scene as u8)); self.next = (scene + 1) % fw::SCENES; @@ -200,16 +236,16 @@ pub(crate) struct Cursors { pub(crate) read: u32, } impl Cursors { - fn valid(self) -> bool { + fn valid(self, capacity: usize) -> bool { [self.total, self.committed, self.read] .into_iter() - .all(|v| (v as usize) < cfg::TVB_BLOCK_SLOTS) + .all(|v| (v as usize) < capacity) } /// Firmware-paused growth permits consumption of the already committed /// prefix. Retired-host growth requires the complete snapshot unchanged. - pub(crate) fn permits(self, after: Self, firmware_request: bool) -> bool { - if !self.valid() - || !after.valid() + fn permits_capacity(self, after: Self, firmware_request: bool, capacity: usize) -> bool { + if !self.valid(capacity) + || !after.valid(capacity) || self.total != after.total || self.committed != after.committed { @@ -218,7 +254,7 @@ impl Cursors { if !firmware_request { return self == after; } - let capacity = cfg::TVB_BLOCK_SLOTS as u32; + let capacity = capacity as u32; (after.read + capacity - self.read) % capacity <= (self.committed + capacity - self.read) % capacity } @@ -227,23 +263,25 @@ impl Cursors { /// Checked block-table indices for one bulk allocation. One ring slot stays /// empty even when all physical backing has been retained by the owner. pub(crate) struct Growth { + capacity: usize, before: Cursors, added: usize, retained: usize, target: usize, } impl Growth { - pub(crate) fn new(before: Cursors, added: usize) -> Result { - if !before.valid() || before.total != before.committed || added == 0 { + fn with_capacity(before: Cursors, added: usize, capacity: usize) -> Result { + if capacity < 2 || capacity > u32::MAX as usize + || !before.valid(capacity) || before.total != before.committed || added == 0 { return Err(EINVAL); } - let retained = (before.total as usize + cfg::TVB_BLOCK_SLOTS - before.read as usize) - % cfg::TVB_BLOCK_SLOTS; - if retained.checked_add(added).ok_or(EOVERFLOW)? >= cfg::TVB_BLOCK_SLOTS { + let retained = (before.total as usize + capacity - before.read as usize) % capacity; + if retained.checked_add(added).ok_or(EOVERFLOW)? >= capacity { return Err(ENOSPC); } - let target = (before.total as usize + added) % cfg::TVB_BLOCK_SLOTS; + let target = (before.total as usize + added) % capacity; Ok(Self { + capacity, before, added, retained, @@ -263,32 +301,25 @@ impl Growth { if index >= self.added { return Err(EINVAL); } - Ok((self.before.total as usize + index) % cfg::TVB_BLOCK_SLOTS) + Ok((self.before.total as usize + index) % self.capacity) } pub(crate) fn available_slot(&self, index: usize) -> Result { if index >= self.available() { return Err(EINVAL); } - Ok((self.before.read as usize + index) % cfg::TVB_BLOCK_SLOTS) + Ok((self.before.read as usize + index) % self.capacity) } } -pub(crate) fn firmware_growth_target(current: usize) -> usize { - current - .saturating_add(fw::GROW_BLOCKS) - .min(cfg::TVB_MAX_BLOCKS) -} - /// The high-water sample estimates capacity, not current occupancy. A growth /// reserves three times the demand only once twice that demand no longer fits. -pub(crate) fn automatic_growth_target(used_pages: usize, current: usize) -> Option { - let maximum = fw::NORMAL_MAX_BLOCKS; +fn automatic_target_for(used_pages: usize, current: usize, pages: usize, maximum: usize) -> Option { if current >= maximum { return None; } let need = used_pages .saturating_mul(2) - .div_ceil(cfg::TVB_PAGES_PER_BLOCK) + .div_ceil(pages) .min(maximum); if need <= current { return None; @@ -296,7 +327,7 @@ pub(crate) fn automatic_growth_target(used_pages: usize, current: usize) -> Opti Some( used_pages .saturating_mul(3) - .div_ceil(cfg::TVB_PAGES_PER_BLOCK) + .div_ceil(pages) .min(maximum), ) } @@ -340,7 +371,11 @@ pub(crate) struct TvbRelease { /// Packed 128-KiB data blocks mapped at guarded 160-KiB virtual strides. /// Firmware owns their contents; CPU zeroing occurs only on fresh backing. +/// +/// M5 blocks use their own geometry ([`BlockGeometry::m5`]): sparse, at the +/// geometry's backing stride, in the M5 compact window. pub(crate) struct Tvb { + geometry: BlockGeometry, addresses: KVec, // Retain sparse-block backing until the pool and its aliases retire. _primary: KernelObject, @@ -352,11 +387,27 @@ impl Tvb { fn bytes(blocks: usize) -> Result { blocks.checked_mul(cfg::TVB_BLOCK_SIZE).ok_or(EOVERFLOW) } + fn backing_bytes(geometry: BlockGeometry, blocks: usize) -> Result { + if geometry == BlockGeometry::NEO { + return Self::bytes(blocks); + } + geometry.backing_bytes(blocks) + } fn map_block( object: &KernelObject, vm: &mmu::Vm, relative: usize, + geometry: BlockGeometry, ) -> Result { + if geometry != BlockGeometry::NEO { + let start = relative.checked_mul(geometry.backing_stride).ok_or(EOVERFLOW)?; + let source = start..start.checked_add(geometry.block_bytes).ok_or(EOVERFLOW)?; + let guard = geometry.stride - geometry.block_bytes; + let mapping = object.map_range(vm, source, geometry.base..geometry.end, + BlockGeometry::PAGE as u64, mmu::PROT_GPU_SHARED_RW, guard)?; + geometry.compact_page(mapping.iova())?; + return Ok(mapping); + } let start = relative.checked_mul(cfg::TVB_BLOCK_SIZE).ok_or(EOVERFLOW)?; let source = start..start.checked_add(cfg::TVB_BLOCK_SIZE).ok_or(EOVERFLOW)?; let mapping = compact_alias( @@ -375,19 +426,29 @@ impl Tvb { vm: &mmu::Vm, blocks: usize, ) -> Result<(Self, KVec)> { - if blocks == 0 || blocks > cfg::TVB_MAX_BLOCKS { + Self::with_geometry(dev, vm, blocks, BlockGeometry::NEO) + } + pub(crate) fn with_geometry( + dev: &AsahiDrmDevice, + vm: &mmu::Vm, + blocks: usize, + geometry: BlockGeometry, + ) -> Result<(Self, KVec)> { + if blocks == 0 || blocks > geometry.max_blocks() { return Err(ERANGE); } - let backing = KernelObject::backing(dev, Self::bytes(blocks)?, CpuMap::WriteCombined)?; + let backing = KernelObject::backing(dev, Self::backing_bytes(geometry, blocks)?, + CpuMap::WriteCombined)?; let mut mappings = KVec::with_capacity(blocks, GFP_KERNEL)?; let mut addresses = KVec::with_capacity(blocks, GFP_KERNEL)?; for index in 0..blocks { - let mapping = Self::map_block(&backing, vm, index)?; + let mapping = Self::map_block(&backing, vm, index, geometry)?; addresses.push(mapping.iova(), GFP_KERNEL)?; mappings.push(mapping, GFP_KERNEL)?; } Ok(( Self { + geometry, addresses, _primary: backing, extensions: KVec::new(), @@ -423,17 +484,18 @@ impl Tvb { if target <= self.blocks() { return Ok(None); } - if target > cfg::TVB_MAX_BLOCKS { + if target > self.geometry.max_blocks() { return Err(ERANGE); } let added = target - self.blocks(); self.extensions.reserve(1, GFP_KERNEL)?; self.addresses.reserve(added, GFP_KERNEL)?; - let backing = KernelObject::backing(dev, Self::bytes(added)?, CpuMap::WriteCombined)?; + let backing = KernelObject::backing(dev, Self::backing_bytes(self.geometry, added)?, + CpuMap::WriteCombined)?; let mut mappings = KVec::with_capacity(added, GFP_KERNEL)?; let mut addresses = KVec::with_capacity(added, GFP_KERNEL)?; for relative in 0..added { - let mapping = Self::map_block(&backing, vm, relative)?; + let mapping = Self::map_block(&backing, vm, relative, self.geometry)?; addresses.push(mapping.iova(), GFP_KERNEL)?; mappings.push(mapping, GFP_KERNEL)?; } @@ -449,10 +511,17 @@ use super::object::Allocator; use core::sync::atomic::{fence, AtomicU32, AtomicU64, Ordering}; use kernel::sync::Arc; +/// Bytes per scene of an M5 metrics entry (the M5 graph layout's metric stride). +const M5_METRIC_STRIDE: usize = 64; + /// First-free metrics entries are shared by all parameter-buffer graphs. /// An entry returns only when the graph and its firmware references retire. pub(crate) struct MetricsIds { used: [AtomicU64; 2], + scene_stride: usize, + /// Bytes from one entry to the next. An M5 entry starts its own pages, so a + /// client VM maps the entries of its own renders and no others. + entry_pitch: usize, } impl MetricsIds { pub(crate) fn new() -> Self { @@ -461,29 +530,86 @@ impl MetricsIds { AtomicU64::new(0), AtomicU64::new(!((1u64 << (fw::METRICS_SLOTS - 64)) - 1)), ], + scene_stride: size_of::(), + entry_pitch: size_of::() * fw::SCENES, } } + /// The allocator of the firmware's metrics entries: G17P's fixed entries, + /// or M5's `M5_METRIC_STRIDE`-byte scene entries. + pub(crate) fn for_firmware(m5: bool) -> Result { + if m5 { + Self::with_stride(M5_METRIC_STRIDE) + } else { + Ok(Self::new()) + } + } + /// The M5 metrics entries: `scene_stride` bytes per scene, bounded by the + /// backing each lease is taken from. + pub(crate) fn with_stride(scene_stride: usize) -> Result { + if scene_stride < 4 || scene_stride % 4 != 0 { return Err(EINVAL); } + let entry_pitch = scene_stride.checked_mul(fw::SCENES) + .and_then(|bytes| bytes.checked_next_multiple_of(mmu::UAT_PGSZ)) + .ok_or(EOVERFLOW)?; + Ok(Self { + used: [AtomicU64::new(0), AtomicU64::new(0)], + scene_stride, + entry_pitch, + }) + } + /// Backing required for every entry owned by this allocator. + pub(crate) fn storage_bytes(&self) -> Result { + self.entry_pitch.checked_mul(self.used.len() * u64::BITS as usize) + .ok_or(EOVERFLOW) + } } pub(crate) struct MetricsLease { - _backing: Arc, + backing: Arc, allocator: Arc, slot: usize, addresses: fw::MetricsAddresses, + scene_stride: usize, + /// M5: the client VM's view of this entry's pages alone. + client: Option, } impl MetricsLease { + /// M5: maps this entry's pages, and no other entry, into the client VM, + /// where the GPU writes the entry while the render runs. + pub(crate) fn map_client(&mut self, vm: &mmu::Vm) -> Result { + if self.client.is_some() { + return Err(EINVAL); + } + let pitch = self.allocator.entry_pitch; + let offset = self.slot.checked_mul(pitch).ok_or(EOVERFLOW)?; + let client = self.backing.map_range(vm, offset..offset + pitch, BlockGeometry::M5_COMPACT, + mmu::UAT_PGSZ as u64, mmu::PROT_GPU_SHARED_RW, 0)?; + self.addresses.client = client.iova(); + self.client = Some(client); + Ok(()) + } /// Clears only the selected retired metrics entry before reusing it. pub(crate) fn new( ids: &Arc, object: &Arc, bases: fw::MetricsAddresses, ) -> Result { + let scene_stride = ids.scene_stride; + let bytes = ids.entry_pitch; + // The fixed G17P entries are bounded by their reserved bits; an M5 + // allocator by the backing the lease is taken from. + let slots = if scene_stride == size_of::() { ids.used.len() * 64 } else { + // The M5 backing covers every entry of its allocator. + if object.size() < ids.storage_bytes()? { + return Err(EINVAL); + } + (object.size() / bytes).min(ids.used.len() * 64) + }; let mut selected = None; for (bank, used) in ids.used.iter().enumerate() { let mut value = used.load(Ordering::Acquire); loop { let bit = value.trailing_ones(); - if bit == 64 { + if bit == 64 || bank * 64 + bit as usize >= slots { break; } match used.compare_exchange_weak( @@ -507,21 +633,24 @@ impl MetricsLease { // Construct the lease before any fallible address/view validation so // every constructor failure returns the selected bit. let mut lease = Self { - _backing: object.clone(), + backing: object.clone(), allocator: ids.clone(), slot, addresses: fw::MetricsAddresses::default(), + scene_stride, + client: None, }; - let offset = slot * fw::METRICS_STRIDE; + let offset = slot * bytes; lease.addresses = fw::MetricsAddresses { firmware: bases.firmware.checked_add(offset as u64).ok_or(EOVERFLOW)?, client: bases.client.checked_add(offset as u64).ok_or(EOVERFLOW)?, }; - let words = object.pointer(offset, fw::METRICS_STRIDE)?.cast::(); + let words = object.pointer(offset, bytes)?.cast::(); object.word(offset)?; - for index in 0..fw::SCENES { - // SAFETY: The complete naturally aligned metrics entry is inside - // the retained object; the exclusive lease excludes other owners. + for index in 0..bytes / size_of::() { + // SAFETY: The complete naturally aligned metrics entry, with the rest of + // its pages on M5, is inside the retained object; the exclusive lease + // excludes other owners. unsafe { words.add(index).write_volatile(0) }; } Ok(lease) @@ -529,6 +658,8 @@ impl MetricsLease { } impl Drop for MetricsLease { fn drop(&mut self) { + // The client's view is unmapped before another render can lease the entry. + drop(self.client.take()); self.allocator.used[self.slot / 64] .fetch_and(!(1u64 << (self.slot % 64)), Ordering::Release); } @@ -548,9 +679,12 @@ pub(crate) struct ManagerArgs { pub(crate) struct Manager { _page_list: mmu::KernelMapping, graph: KernelObject, + layout: layout::Layout, tvb: Tvb, metrics: MetricsLease, - scenes: Scenes, + /// Boxed: the per-scene owner table would otherwise make every `Manager` value large, and + /// one is built and moved through the frames of the submit path that creates a pair. + scenes: KBox, page_list_client: u64, scratch: u64, buffer_id: u8, @@ -616,41 +750,56 @@ impl Manager { metrics: MetricsLease, args: ManagerArgs, ) -> Result { + let geometry = tvb.geometry; + let neo = geometry == BlockGeometry::NEO; + let layout = if neo { layout::Layout::NEO } else { layout::Layout::m5(geometry)? }; + if metrics.scene_stride != layout.metric_stride { + return Err(EINVAL); + } let mut graph = alloc.kernel( - fw::SIZE, + layout.bytes, cfg::TVB_PAGE, mmu::PROT_GPU_FW_SHARED_RW, CpuMap::WriteCombined, )?; - let source = fw::PAGE_LIST..fw::PAGE_LIST + fw::PAGE_LIST_BYTES; - let page_list = compact_alias( - &graph, - vm, - source, - cfg::TVB_PAGE, - mmu::PROT_GPU_FW_SHARED_RW, - mmu::UAT_PGSZ, - )?; + let source = layout.pages..layout.pages + layout.page_bytes; + let page_list = if neo { + compact_alias( + &graph, + vm, + source, + cfg::TVB_PAGE, + mmu::PROT_GPU_FW_SHARED_RW, + mmu::UAT_PGSZ, + )? + } else { + graph.map_range(vm, source, geometry.base..geometry.end, + cfg::TVB_PAGE, mmu::PROT_GPU_FW_SHARED_RW, mmu::UAT_PGSZ)? + }; let address = graph.gpu_va(); - graph.initialize::(0, |view| { - view.init(&fw::GraphArgs { - fw_va: address, - page_list_client: page_list.iova(), - metrics: metrics.addresses, - blocks: tvb.addresses(), - scratch: args.scratch, - discard: args.discard, - buffer_id: args.buffer_id.into(), - generation: args.generation, - }) - })?; + let graph_args = fw::GraphArgs { + fw_va: address, + page_list_client: page_list.iova(), + metrics: metrics.addresses, + blocks: tvb.addresses(), + scratch: args.scratch, + discard: args.discard, + buffer_id: args.buffer_id.into(), + generation: args.generation, + }; + if neo { + graph.initialize::(0, |view| view.init(&graph_args))?; + } else { + layout.initialize(geometry, &graph_args, graph.bytes_mut())?; + } fence(Ordering::SeqCst); - let mut scenes = Scenes::new(); + let mut scenes = KBox::new(Scenes::with_slots(layout.scratch_slots)?, GFP_KERNEL)?; scenes.commit(args.first_ordinal, fw::FIRST_SCENE)?; let page_list_client = page_list.iova(); Ok(Self { _page_list: page_list, graph, + layout, tvb, metrics, scenes, @@ -677,6 +826,13 @@ impl Manager { pub(crate) fn blocks(&self) -> usize { self.tvb.blocks() } + pub(crate) fn page_count(&self) -> Result { + self.blocks().checked_mul(self.tvb.geometry.pages()).ok_or(EOVERFLOW)? + .try_into().map_err(|_| EOVERFLOW) + } + pub(crate) fn firmware_target(&self) -> usize { + self.blocks().saturating_add(fw::GROW_BLOCKS).min(self.tvb.geometry.max_blocks()) + } /// Read-only diagnostic while firmware is halted. In particular this /// does not reset the high-water counter as automatic_target() does. pub(crate) fn report_recovery(&self) { @@ -689,8 +845,8 @@ impl Manager { read(core::mem::offset_of!(fw::State, block_capacity)), read(core::mem::offset_of!(fw::State, producer)), read(core::mem::offset_of!(fw::State, consumer)), - read(fw::CONTROL + core::mem::offset_of!(fw::Control, max_pages)), - read(fw::CONTROL + core::mem::offset_of!(fw::Control, reset)), + read(self.layout.control + core::mem::offset_of!(fw::Control, max_pages)), + read(self.layout.control + core::mem::offset_of!(fw::Control, reset)), self.initialized); } pub(crate) fn state_va(&self) -> u64 { @@ -700,20 +856,33 @@ impl Manager { self.page_list_client } pub(crate) fn trailer_va(&self) -> u64 { - self.graph.gpu_va() + fw::SCENE_TABLE as u64 + (fw::SCENES * size_of::()) as u64 + self.graph.gpu_va() + self.layout.scenes as u64 + (fw::SCENES * size_of::()) as u64 } pub(crate) fn scene_va(&self, index: usize) -> Result { if index >= fw::SCENES { return Err(EINVAL); } - Ok(self.graph.gpu_va() + fw::SCENE_TABLE as u64 + (index * size_of::()) as u64) + Ok(self.graph.gpu_va() + self.layout.scenes as u64 + (index * size_of::()) as u64) } pub(crate) fn scene_registers(&self, index: usize) -> Result<(u64, u64)> { - fw::scene_registers(index, self.scratch, self.metrics.addresses.client) + if self.tvb.geometry == BlockGeometry::NEO { + return fw::scene_registers(index, self.scratch, self.metrics.addresses.client); + } + if index >= fw::SCENES { return Err(EINVAL); } + let scratch = self.scratch.checked_sub(self.tvb.geometry.base).ok_or(EINVAL)? + .checked_add((index % self.layout.scratch_slots * self.layout.scratch_stride) as u64) + .ok_or(EOVERFLOW)?; + let metric = self.metrics.addresses.client.checked_sub(self.tvb.geometry.base) + .ok_or(EINVAL)? + .checked_add((index * self.layout.metric_stride) as u64).ok_or(EOVERFLOW)?; + if scratch > u32::MAX as u64 || metric > u32::MAX as u64 || metric & 3 != 0 { + return Err(EINVAL); + } + Ok((scratch, metric | 1)) } pub(crate) fn prepare_scene(&mut self, ordinal: u64) -> Result { let scene = self.scenes.select(ordinal)?; - let offset = fw::SCENE_TABLE + scene * size_of::(); + let offset = self.layout.scenes + scene * size_of::(); let used = self .graph .word(offset + core::mem::offset_of!(fw::Scene, used))?; @@ -738,7 +907,7 @@ impl Manager { self.scenes.release(ordinal) } pub(crate) fn prepare_submission(&mut self) -> Result> { - let word = self.graph.word(fw::COUNTER)?; + let word = self.graph.word(self.layout.counter)?; Ok(Submitted { word, next: word.load(Ordering::Relaxed).wrapping_add(1), @@ -749,10 +918,10 @@ impl Manager { fn sample_max_pages(&mut self) -> Result { let max_pages = self .graph - .word(fw::CONTROL + core::mem::offset_of!(fw::Control, max_pages))?; + .word(self.layout.control + core::mem::offset_of!(fw::Control, max_pages))?; let reset = self .graph - .word(fw::CONTROL + core::mem::offset_of!(fw::Control, reset))?; + .word(self.layout.control + core::mem::offset_of!(fw::Control, reset))?; fence(Ordering::Acquire); let pages = max_pages.load(Ordering::Relaxed); fence(Ordering::Release); @@ -764,7 +933,8 @@ impl Manager { pub(crate) fn automatic_target(&mut self) -> Result> { self.sample_max_pages()?; let pages = core::mem::take(&mut self.shrink.growth_max); - Ok(automatic_growth_target(pages as usize, self.blocks())) + Ok(automatic_target_for(pages as usize, self.blocks(), self.tvb.geometry.pages(), + self.layout.normal_blocks)) } /// Records the blocks a pass layout requires, the floor of any shrink. /// A pass is being prepared at `now`; the heap counts as busy. @@ -782,12 +952,13 @@ impl Manager { self.tvb.primary_blocks().max(self.shrink.required).max(recent) } /// Whether the heap holds growth beyond its primary backing. + /// M5 heaps are never shrunk. pub(crate) fn grown(&self) -> bool { - !self.tvb.extensions.is_empty() + self.tvb.geometry == BlockGeometry::NEO && !self.tvb.extensions.is_empty() } /// Whether the heap holds growth that a shrink could give back. pub(crate) fn shrinkable(&self) -> bool { - self.tvb.extensions.last().is_some_and(|extension| { + self.tvb.geometry == BlockGeometry::NEO && self.tvb.extensions.last().is_some_and(|extension| { self.blocks() - Tvb::extension_blocks(extension) >= self.shrink_floor() }) } @@ -836,9 +1007,9 @@ impl Manager { } fn cursors(&self) -> Result { Ok(Cursors { - total: self.graph.word(fw::CONTROL)?.load(Ordering::Relaxed), - committed: self.graph.word(fw::CONTROL + 4)?.load(Ordering::Relaxed), - read: self.graph.word(fw::CONTROL + 8)?.load(Ordering::Relaxed), + total: self.graph.word(self.layout.control)?.load(Ordering::Relaxed), + committed: self.graph.word(self.layout.control + 4)?.load(Ordering::Relaxed), + read: self.graph.word(self.layout.control + 8)?.load(Ordering::Relaxed), }) } } @@ -861,12 +1032,14 @@ impl Manager { /// Validate every retained sparse block and its reserved trailing guard in /// one VM snapshot before constructing more aliases. pub(crate) fn validate_vm(&self, vm: &mmu::Vm) -> Result { - if self.blocks() < cfg::TVB_INITIAL_PAGES.len() - || self.blocks() > cfg::TVB_MAX_BLOCKS + let geometry = self.tvb.geometry; + let minimum = if geometry == BlockGeometry::NEO { cfg::TVB_INITIAL_PAGES.len() } else { 1 }; + if self.blocks() < minimum + || self.blocks() > geometry.max_blocks() || !vm.covers_sparse_blocks_with_guards( self.tvb.addresses(), - cfg::TVB_BLOCK_SIZE as u64, - cfg::TVB_BLOCK_STRIDE, + geometry.block_bytes as u64, + geometry.stride as u64, ) { return Err(EFAULT); @@ -893,23 +1066,23 @@ impl Manager { return Ok(None); }; let before = self.cursors()?; - let plan = Growth::new(before, target - previous)?; + let plan = Growth::with_capacity(before, target - previous, self.tvb.geometry.capacity)?; let firmware_request = descriptor.is_none(); let pages = target - .checked_mul(cfg::TVB_PAGES_PER_BLOCK) + .checked_mul(self.tvb.geometry.pages()) .ok_or(EOVERFLOW)? as u32; let mut added = KVec::with_capacity(growth.addresses.len(), GFP_KERNEL)?; for (index, &address) in growth.addresses.iter().enumerate() { - let first = fw::compact_page(address)?; + let first = self.tvb.geometry.compact_page(address)?; first - .checked_add(cfg::TVB_PAGES_PER_BLOCK as u32 - 1) + .checked_add(self.tvb.geometry.pages() as u32 - 1) .ok_or(EOVERFLOW)?; added.push((plan.added_slot(index)?, first), GFP_KERNEL)?; } let rebuilt_count = if firmware_request { 0 } else { - plan.available() * cfg::TVB_PAGES_PER_BLOCK + plan.available() * self.tvb.geometry.pages() }; let mut rebuilt = KVec::with_capacity(rebuilt_count, GFP_KERNEL)?; if !firmware_request { @@ -920,15 +1093,15 @@ impl Manager { let first = if ordinal < plan.retained() { let slot = plan.available_slot(ordinal)?; self.graph - .word(fw::BLOCK_TABLE + slot * size_of::())? + .word(self.layout.blocks + slot * size_of::())? .load(Ordering::Relaxed) } else { added[ordinal - plan.retained()].1 }; first - .checked_add(cfg::TVB_PAGES_PER_BLOCK as u32 - 1) + .checked_add(self.tvb.geometry.pages() as u32 - 1) .ok_or(EOVERFLOW)?; - for page in 0..cfg::TVB_PAGES_PER_BLOCK { + for page in 0..self.tvb.geometry.pages() { rebuilt.push(first + page as u32, GFP_KERNEL)?; } } @@ -962,7 +1135,7 @@ impl Manager { return Err(EBUSY); } let firmware_request = candidate.descriptor.is_none(); - let base = self.graph.pointer(0, fw::SIZE)?.cast::(); + let base = self.graph.pointer(0, self.layout.bytes)?.cast::(); // Bounds, actual alignment and every reserved host entry are checked // before transferring aliases or consuming the caller-owned plan. self.graph.word(0)?; @@ -972,7 +1145,8 @@ impl Manager { { return Err(EIO); } - if !candidate.before.permits(self.cursors()?, firmware_request) { + if !candidate.before.permits_capacity(self.cursors()?, firmware_request, + self.tvb.geometry.capacity) { return Err(EAGAIN); } // The callback retains its input on error. The caller can therefore @@ -1013,13 +1187,13 @@ impl Manager { unsafe { AtomicU32::from_ptr(base.add(offset / 4)).store(value, Ordering::Relaxed) }; }; for (slot, first) in added.into_iter() { - let at = fw::BLOCK_TABLE + slot * size_of::(); + let at = self.layout.blocks + slot * size_of::(); write(at, first); write(at + 4, 0); } if descriptor.is_some() { for (entry, value) in rebuilt.into_iter().enumerate() { - write(fw::PAGE_LIST + entry * 4, value); + write(self.layout.pages + entry * 4, value); } write(core::mem::offset_of!(fw::State, pages), pages); write(core::mem::offset_of!(fw::State, producer), plan.target()); @@ -1027,9 +1201,9 @@ impl Manager { write(core::mem::offset_of!(fw::State, last_page), pages - 1); } append_reserved(&mut self.tvb.extensions, extension); - write(fw::CONTROL, plan.target()); + write(self.layout.control, plan.target()); fence(Ordering::SeqCst); - write(fw::CONTROL + 4, plan.target()); + write(self.layout.control + 4, plan.target()); fence(Ordering::SeqCst); Ok(descriptor) } @@ -1069,7 +1243,7 @@ impl Manager { let before = self.cursors()?; let retained = (before.total as usize + cfg::TVB_BLOCK_SLOTS - before.read as usize) % cfg::TVB_BLOCK_SLOTS; - if !before.valid() || before.total != before.committed || retained != blocks { + if !before.valid(cfg::TVB_BLOCK_SLOTS) || before.total != before.committed || retained != blocks { return Ok(None); } // Allocation failure only postpones the shrink to a later pass. @@ -1158,8 +1332,10 @@ impl BufferIds { } impl MetricsIds { pub(crate) fn used(&self) -> u32 { + let reserved = if self.scene_stride == size_of::() { 128 - fw::METRICS_SLOTS as u32 } + else { 0 }; self.used.iter().map(|bits| bits.load(Ordering::Acquire).count_ones()).sum::() - - (128 - fw::METRICS_SLOTS as u32) + - reserved } } diff --git a/drivers/gpu/drm/asahi/g17/buffer/geometry.rs b/drivers/gpu/drm/asahi/g17/buffer/geometry.rs new file mode 100644 index 0000000000000..c871f470c1a15 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/buffer/geometry.rs @@ -0,0 +1,79 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Parameter-buffer backing geometry. Wire records carry each block's address, +//! so allocation stride and the reserved trailing guard are host policy. + +use crate::hw::t8140::resources as cfg; +use kernel::prelude::*; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct BlockGeometry { + pub(crate) base: u64, + pub(crate) end: u64, + pub(crate) block_bytes: usize, + pub(crate) backing_stride: usize, + pub(crate) stride: usize, + pub(crate) capacity: usize, +} + +impl BlockGeometry { + pub(crate) const PAGE: usize = 0x8000; + /// M5 parameter resources use 32-bit offsets from this compact base. + pub(crate) const M5_COMPACT: core::ops::Range = 0x10_0000_0000..0x11_0000_0000; + pub(crate) const NEO: Self = Self { + base: cfg::USER_BASE, + end: cfg::USER_END, + block_bytes: cfg::TVB_BLOCK_SIZE, + backing_stride: cfg::TVB_BLOCK_STRIDE as usize, + stride: cfg::TVB_BLOCK_STRIDE as usize, + capacity: cfg::TVB_BLOCK_SLOTS, + }; + + pub(crate) fn m5(memory_bytes: u64, initial_bytes: u64) -> Result { + let capacity = crate::g17::fw::m5::pool::parameter_capacity( + memory_bytes, initial_bytes, + ).map_err(|_| EINVAL)?; + // Retain one unallocated inventory slot, as in the shared ring manager. + if capacity < 2 || capacity.checked_mul(16).is_none_or(|n| n > 0x3f_ffff) { + return Err(EINVAL); + } + Ok(Self { + base: Self::M5_COMPACT.start, + end: Self::M5_COMPACT.end, + block_bytes: 0x80000, + backing_stride: 0x80000, + stride: 0x88000, + capacity, + }) + } + + pub(crate) const fn pages(self) -> usize { self.block_bytes / Self::PAGE } + pub(crate) const fn max_blocks(self) -> usize { self.capacity - 1 } + + pub(crate) fn validate(self) -> Result { + if self.capacity < 2 || self.capacity > u32::MAX as usize + || self.base >= self.end || self.base % Self::PAGE as u64 != 0 + || self.block_bytes == 0 || self.block_bytes % Self::PAGE != 0 + || self.backing_stride < self.block_bytes || self.backing_stride % Self::PAGE != 0 + || self.stride <= self.block_bytes || self.stride % Self::PAGE != 0 { + return Err(EINVAL); + } + Ok(()) + } + + pub(crate) fn backing_bytes(self, blocks: usize) -> Result { + self.validate()?; + if blocks == 0 || blocks > self.max_blocks() { return Err(EINVAL); } + (blocks - 1).checked_mul(self.backing_stride) + .and_then(|n| n.checked_add(self.block_bytes)).ok_or(EOVERFLOW) + } + + pub(crate) fn compact_page(self, address: u64) -> Result { + let relative = address.checked_sub(self.base).ok_or(EINVAL)?; + if relative % Self::PAGE as u64 != 0 + || address.checked_add(self.block_bytes as u64).ok_or(EOVERFLOW)? > self.end { + return Err(EINVAL); + } + (relative / Self::PAGE as u64).try_into().map_err(|_| EOVERFLOW) + } +} diff --git a/drivers/gpu/drm/asahi/g17/buffer/layout.rs b/drivers/gpu/drm/asahi/g17/buffer/layout.rs new file mode 100644 index 0000000000000..ca4ad54750658 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/buffer/layout.rs @@ -0,0 +1,105 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Wire geometry for the shared parameter-buffer owner and growth machinery. + +use super::{fw, BlockGeometry}; +use kernel::prelude::*; + +#[derive(Clone, Copy)] +pub(super) struct Layout { + pub(super) pages: usize, + pub(super) page_bytes: usize, + pub(super) control: usize, + pub(super) counter: usize, + pub(super) scenes: usize, + pub(super) blocks: usize, + pub(super) bytes: usize, + pub(super) scratch_slots: usize, + pub(super) scratch_stride: usize, + pub(super) metric_stride: usize, + pub(super) normal_blocks: usize, +} + +fn align(value: usize, alignment: usize) -> Result { + value.checked_add(alignment - 1).map(|n| n & !(alignment - 1)).ok_or(EOVERFLOW) +} + +impl Layout { + pub(super) const NEO: Self = Self { + pages: fw::PAGE_LIST, page_bytes: fw::PAGE_LIST_BYTES, + control: fw::CONTROL, counter: fw::COUNTER, scenes: fw::SCENE_TABLE, + blocks: fw::BLOCK_TABLE, bytes: fw::SIZE, + scratch_slots: fw::SCRATCH_SLOTS, scratch_stride: fw::SCRATCH_STRIDE as usize, + metric_stride: 4, normal_blocks: fw::NORMAL_MAX_BLOCKS, + }; + + pub(super) fn m5(geometry: BlockGeometry) -> Result { + geometry.validate()?; + let blocks = 0x4000; + let pages = align(geometry.capacity.checked_mul(8) + .and_then(|n| n.checked_add(blocks)).ok_or(EOVERFLOW)?, 0x4000)?; + let page_bytes = align(geometry.capacity.checked_mul(geometry.pages()) + .and_then(|n| n.checked_mul(4)).ok_or(EOVERFLOW)?, 0x4000)?; + let scratch_stride = crate::g17::fw::m5::pool::parameter_scratch_stride( + geometry.capacity, 0).map_err(|_| EINVAL)?; + Ok(Self { + pages, page_bytes, control: 0x300, counter: 0x200, + scenes: 0x400, blocks, bytes: pages.checked_add(page_bytes).ok_or(EOVERFLOW)?, + scratch_slots: fw::SCENES, scratch_stride, metric_stride: 64, + normal_blocks: geometry.capacity / 3, + }) + } + + pub(super) fn initialize(self, geometry: BlockGeometry, args: &fw::GraphArgs<'_>, + out: &mut [u8]) -> Result { + geometry.validate()?; + if out.len() < self.bytes || args.generation == 0 || args.blocks.is_empty() + || args.blocks.len() > geometry.max_blocks() { return Err(EINVAL); } + let scratch = args.scratch.checked_sub(geometry.base).ok_or(EINVAL)?; + let discard = args.discard.checked_sub(geometry.base).ok_or(EINVAL)?; + let scratch_end = scratch.checked_add((self.scratch_slots * self.scratch_stride) as u64) + .ok_or(EOVERFLOW)?; + if scratch_end > u32::MAX as u64 || discard > u32::MAX as u64 { return Err(EINVAL); } + let pages = u32::try_from(args.blocks.len().checked_mul(geometry.pages()).ok_or(EOVERFLOW)?)?; + let maximum = u32::try_from(geometry.capacity.checked_mul(geometry.pages()).ok_or(EOVERFLOW)?)?; + let normal = u32::try_from(self.normal_blocks.checked_mul(geometry.pages()).ok_or(EOVERFLOW)?)?; + // Every object is unpublished here. Keep all unused words zero on reuse. + out[..self.bytes].fill(0); + fn put(out: &mut [u8], at: usize, value: [u8; N]) -> Result { + out.get_mut(at..at.checked_add(N).ok_or(EOVERFLOW)?).ok_or(EINVAL)? + .copy_from_slice(&value); + Ok(()) + } + let address = |offset: usize| args.fw_va.checked_add(offset as u64).ok_or(EOVERFLOW); + for (index, &block) in args.blocks.iter().enumerate() { + let first = geometry.compact_page(block)?; + put(out, self.blocks + index * 8, first.to_le_bytes())?; + for page in 0..geometry.pages() { + put(out, self.pages + (index * geometry.pages() + page) * 4, + first.checked_add(page as u32).ok_or(EOVERFLOW)?.to_le_bytes())?; + } + } + for index in 0..fw::SCENES { + let at = self.scenes + index * size_of::(); + let metric = (index * self.metric_stride) as u64; + put(out, at, args.metrics.client.checked_add(metric).ok_or(EOVERFLOW)?.to_le_bytes())?; + put(out, at + 8, args.metrics.firmware.checked_add(metric).ok_or(EOVERFLOW)?.to_le_bytes())?; + put(out, at + 0x28, (scratch + (index % self.scratch_slots * self.scratch_stride) as u64).to_le_bytes())?; + put(out, at + 0x40, address(self.control + 0x40)?.to_le_bytes())?; + } + put(out, self.scenes + fw::FIRST_SCENE * size_of::() + 0x4c, args.generation.to_le_bytes())?; + for (at, value) in [ + (0xc, args.buffer_id), (0x30, u32::try_from(self.page_bytes)?), (0x34, pages), + (0x38, u32::try_from(geometry.capacity)?), (0x3c, args.blocks.len() as u32), + (0x54, pages - 1), (0x58, u32::try_from(geometry.block_bytes)?), + (0x7c, maximum), (0x80, normal), (self.control, args.blocks.len() as u32), + (self.control + 4, args.blocks.len() as u32), (self.control + 0x60, 1), + ] { put(out, at, value.to_le_bytes())?; } + for (at, value) in [ + (0x20, address(self.pages)?), (0x28, args.page_list_client), + (0x44, address(self.blocks)?), (0x4c, address(self.control)?), + (0x64, address(self.counter)?), (0x84, discard), + ] { put(out, at, value.to_le_bytes())?; } + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/g17/buffer/target.rs b/drivers/gpu/drm/asahi/g17/buffer/target.rs index bd96ecc9d460e..8561053a336f2 100644 --- a/drivers/gpu/drm/asahi/g17/buffer/target.rs +++ b/drivers/gpu/drm/asahi/g17/buffer/target.rs @@ -39,6 +39,7 @@ const GTP6_SIZE: usize = GTP_SLOTS /// Per-command geometry, independent of the capacity already allocated. #[derive(Copy, Clone, Debug)] pub(crate) struct Layout { + pub(crate) tilemap_size: usize, pub(crate) layer_offset: usize, pub(crate) heap_offset: usize, pub(crate) used: usize, @@ -49,6 +50,8 @@ pub(crate) struct Layout { gtp: [Option; 8], /// Bytes of the TPC object the command uses. tpc_end: usize, + /// The M5 tiler slices of the TPC object and their cluster controls. + m5: Option<(crate::g17::fw::m5::target::TargetSlices, [u64; 3])>, } impl Layout { pub(crate) fn new( @@ -134,6 +137,7 @@ impl Layout { tpc_end = position; } Ok(Self { + tilemap_size, layer_offset, heap_offset, used, @@ -141,9 +145,34 @@ impl Layout { tvb_blocks, gtp, tpc_end, + m5: None, }) } + pub(crate) fn with_m5_slices(mut self, pass: &RenderPass, clusters: u32, groups: u32, + segments: u32, parameter_bytes: usize) -> Result { + let slices = crate::g17::fw::m5::target::TargetSlices::new( + self.tilemap_size, self.tpc_size, pass.layers, groups, segments, parameter_bytes) + .ok_or(EINVAL)?; + // The last word is the part's minimum parameter-buffer size, whatever + // the group count. + let control = crate::g17::fw::m5::target::cluster_control(groups, + crate::g17::fw::m5::target::min_parameter_buffer_pages(clusters)) + .ok_or(EINVAL)?; + self.m5 = Some((slices, control)); + Ok(self) + } + pub(crate) fn is_m5(&self) -> bool { self.m5.is_some() } + fn check_capacity(&self, stride: usize) -> Result { + if self.m5.is_none() && stride > cfg::TARGET_RECORD_MAX { + return Err(ERANGE); + } + stride.checked_mul(DEPTH).ok_or(EOVERFLOW)?; + Ok(()) + } pub(crate) fn tpc_capacity(&self) -> Result { + if let Some((slices, _)) = self.m5 { + return aligned(slices.bytes, mmu::UAT_PGSZ); + } // TPC_MAX bounds the tail-pointer cache itself. On 25G83 this object // also contains GTP sub-buffers, notably 64 times the tile map. Applying // the cache limit to their combined extent rejects valid large targets. @@ -157,7 +186,8 @@ impl Layout { Ok(aligned(self.tpc_end, mmu::UAT_PGSZ)?.max(cfg::TPC_MIN)) } pub(crate) fn record_stride(&self) -> Result { - Ok(aligned(self.used, mmu::UAT_PGSZ)?.max(cfg::TARGET_RECORD_MIN)) + let bytes = aligned(self.used, mmu::UAT_PGSZ)?; + Ok(if self.m5.is_some() { bytes } else { bytes.max(cfg::TARGET_RECORD_MIN) }) } } @@ -167,10 +197,15 @@ struct Object { backing: KernelObject, } impl Object { - fn new(dev: &AsahiDrmDevice, vm: &mmu::Vm, size: usize) -> Result { + fn new(dev: &AsahiDrmDevice, vm: &mmu::Vm, size: usize, m5: bool) -> Result { let backing = KernelObject::backing(dev, size, CpuMap::WriteCombined)?; let prot = mmu::PROT_GPU_FW_SHARED_RW; - let mapping = compact_alias(&backing, vm, 0..size, cfg::TVB_PAGE, prot, mmu::UAT_PGSZ)?; + let mapping = if m5 { + backing.map_range(vm, 0..size, super::BlockGeometry::M5_COMPACT, + cfg::TVB_PAGE, prot, mmu::UAT_PGSZ)? + } else { + compact_alias(&backing, vm, 0..size, cfg::TVB_PAGE, prot, mmu::UAT_PGSZ)? + }; Ok(Self { address: mapping.iova(), _mapping: mapping, @@ -196,6 +231,7 @@ pub(crate) struct Target { stride: usize, slots: [Option; DEPTH], retired: KVec, + m5: bool, } impl Target { /// Keep this allocation frame separate from the rest of the render graph. @@ -203,11 +239,10 @@ impl Target { pub(crate) fn new(dev: &AsahiDrmDevice, vm: &mmu::Vm, layout: &Layout) -> Result> { let tpc = layout.tpc_capacity()?; let stride = layout.record_stride()?; - if stride > cfg::TARGET_RECORD_MAX { - return Err(ERANGE); - } - let records = Object::new(dev, vm, stride * DEPTH)?; - let tpc = Object::new(dev, vm, tpc)?; + layout.check_capacity(stride)?; + let m5 = layout.m5.is_some(); + let records = Object::new(dev, vm, stride * DEPTH, m5)?; + let tpc = Object::new(dev, vm, tpc, m5)?; KBox::new( Self { records, @@ -215,6 +250,7 @@ impl Target { stride, slots: [None; DEPTH], retired: KVec::new(), + m5, }, GFP_KERNEL, ) @@ -224,6 +260,27 @@ impl Target { self.tpc.address } + /// The M5 tiling words naming the TPC object's slices of `layout`. + pub(crate) fn m5_extra(&self, layout: &Layout) + -> Result { + if !self.m5 { return Err(EINVAL); } + let (slices, control) = layout.m5.ok_or(EINVAL)?; + if slices.bytes > self.tpc.backing.size() { return Err(ERANGE); } + let base = self.tpc.address.checked_sub(super::BlockGeometry::M5_COMPACT.start) + .ok_or(EINVAL)?; + let mut pointers = [0; 8]; + for index in 0..8 { + if slices.sizes[index] != 0 { + pointers[index] = base.checked_add(slices.offsets[index] as u64).ok_or(EOVERFLOW)?; + } + } + Ok(crate::g17::fw::m5::render_registers::TaExtra { + gtp1: pointers[1], gtp2: pointers[2], gtp4: pointers[4], gtp5: pointers[5], + gtp6: pointers[6], gtp7: pointers[7], segment_control: control[0], segment_index: control[1], + min_free_tvb_pages: control[2], scs_bracket: 0, matrix_bracket: 0, + }) + } + /// GPU addresses of the tiler parameter sub-buffers of `layout` (25G83), zero for an empty /// or absent one. The TPC object holds them after the TPC. pub(crate) fn gtp_va(&self, layout: &Layout) -> Result<[u64; 8]> { @@ -243,6 +300,7 @@ impl Target { /// not clear a previous large target's complete retained record capacity. pub(crate) fn prepare(&mut self, ordinal: u64, layout: &Layout) -> Result { let slot = ordinal as usize % DEPTH; + if self.m5 != layout.m5.is_some() { return Err(EINVAL); } if self.slots[slot].is_some_and(|owner| owner != ordinal) { return Err(EBUSY); } @@ -285,15 +343,14 @@ impl Target { ) -> Result { let tpc = layout.tpc_capacity()?; let stride = layout.record_stride()?; + if self.m5 != layout.m5.is_some() { return Err(EINVAL); } + layout.check_capacity(stride)?; let mut grew = false; if tpc > self.tpc.backing.size() { - Self::replace(&mut self.retired, &mut self.tpc, dev, vm, tpc, owners)?; + Self::replace(&mut self.retired, &mut self.tpc, dev, vm, tpc, owners, self.m5)?; grew = true; } if stride > self.stride { - if stride > cfg::TARGET_RECORD_MAX { - return Err(ERANGE); - } Self::replace( &mut self.retired, &mut self.records, @@ -301,6 +358,7 @@ impl Target { vm, stride * DEPTH, owners, + self.m5, )?; self.stride = stride; self.slots = [None; DEPTH]; @@ -315,12 +373,13 @@ impl Target { vm: &mmu::Vm, size: usize, owners: &[Option; DEPTH], + m5: bool, ) -> Result { let retain = owners.iter().any(Option::is_some); if retain { retired.reserve(1, GFP_KERNEL)?; } - let new = Object::new(dev, vm, size)?; + let new = Object::new(dev, vm, size, m5)?; let previous = core::mem::replace(old, new); if retain { super::append_reserved( diff --git a/drivers/gpu/drm/asahi/g17/capacity.rs b/drivers/gpu/drm/asahi/g17/capacity.rs index afc43914ff963..85a09e48c112f 100644 --- a/drivers/gpu/drm/asahi/g17/capacity.rs +++ b/drivers/gpu/drm/asahi/g17/capacity.rs @@ -73,7 +73,7 @@ impl kernel::debugfs::Writer for View { writeln!(f, "execution_contexts_fenced free_waiting={} overruns={}", fenced, overruns)?; let [kept, fallbacks] = super::completion::window_counters(); writeln!(f, "timestamp_window firmware={} host_fallbacks={}", kept, fallbacks)?; - let upper = crate::hw::t8140::dynamic::upper(uat.geometry().kernel_range()); + let upper = uat.kernel_dynamic_range(); for (name, vm, range) in [ ("dynamic_lower_bytes", uat.kernel_lower_vm(), crate::hw::t8140::dynamic::LOWER), ("dynamic_upper_bytes", uat.kernel_vm(), upper), diff --git a/drivers/gpu/drm/asahi/g17/channel.rs b/drivers/gpu/drm/asahi/g17/channel.rs index da02c6bf21da9..d65ed18b6c786 100644 --- a/drivers/gpu/drm/asahi/g17/channel.rs +++ b/drivers/gpu/drm/asahi/g17/channel.rs @@ -58,19 +58,20 @@ enum ReportRing { } impl ReportRing { - fn geometry(self) -> (usize, usize, u32, u32) { + /// State and entry offsets, capacity and the record tags the family allows. + fn geometry(self, init: &InitData) -> (usize, usize, u32, u64) { match self { Self::Trace => ( status::TRACE_STATE as usize, status::TRACE_RING as usize, TRACE_SLOTS, - TRACE_TYPES, + u64::from(TRACE_TYPES), ), Self::Statistics => ( status::STATS_STATE as usize, status::STATS_RING as usize, STATISTICS_SLOTS, - STATISTICS_TYPES, + init.statistics_types(), ), } } @@ -243,7 +244,7 @@ impl Rings { accept_statistics: bool, statistics_cutoff: &mut Option, ) -> Result { - let (state, entries, capacity, allowed_types) = ring.geometry(); + let (state, entries, capacity, allowed_types) = ring.geometry(init); let consumer_word = init.status_word(role, state)?; let producer_word = init.status_word(role, state + RX_PRODUCER_OFFSET)?; fence(Ordering::Acquire); @@ -272,12 +273,12 @@ impl Rings { let words = init.status_read_words::(role, offset)?; super::trace::report(role as u8, ring as u8, &words); let tag = words[0]; - if tag >= u32::BITS || allowed_types & (1 << tag) == 0 { + if tag >= u64::BITS || allowed_types & (1u64 << tag) == 0 { return Err(EIO); } if matches!(ring, ReportRing::Statistics) { let admitted = accept_statistics && statistics_cutoff.is_none(); - reports.record(role, words, host_ns, admitted); + reports.record(role, words, host_ns, allowed_types, admitted); if admitted { telemetry.record(&words, host_ns, max_age_ns); } @@ -318,7 +319,7 @@ impl Rings { init.publish_control( producer, record.bytes(), - record.opcode().asserts_gpu_power(), + init.control_asserts_gpu_power(record.opcode()), next, )?; CONTROL_PUBLISHED.fetch_add(1, Ordering::AcqRel); @@ -385,7 +386,7 @@ impl Events { let words = init.status_read_words(role, offset)?; super::trace::report(role as u8, 2, &words); let record = EventRecord::from_words(words); - let event = record.decode_for(self.graph); + let event = record.decode_for(self.graph, init.is_m5()); fence(Ordering::SeqCst); consumer_word.store((consumer + 1) & (EVENT_SLOTS - 1), Ordering::Relaxed); fence(Ordering::SeqCst); diff --git a/drivers/gpu/drm/asahi/g17/command.rs b/drivers/gpu/drm/asahi/g17/command.rs index c65f7de6ac915..796d072364fd2 100644 --- a/drivers/gpu/drm/asahi/g17/command.rs +++ b/drivers/gpu/drm/asahi/g17/command.rs @@ -10,7 +10,8 @@ use super::fw::{ }; use crate::util::{AnyBitPattern, Reader}; use core::ops::Range; -use kernel::{prelude::*, uapi}; +use kernel::{prelude::*, sync::Arc, uapi}; +use super::render_resources::{ComputePrograms, Programs}; const _: () = assert!(uapi::drm_asahi_compute_flags_DRM_ASAHI_COMPUTE_AS_BUILD == 1 << 2); @@ -29,14 +30,22 @@ const PAGE_SIZE: u64 = 0x4000; const ZLS_METADATA_STRIDE: u64 = 0x80; const MAX_DIMENSION: u16 = 16384; const MAX_LAYERS: u16 = 2048; -const MAX_TILE_BUFFER: u64 = 32768; +pub(crate) const MAX_TILE_BUFFER: u64 = 32768; +// Eight RGBA8 render targets take 64 bytes a pixel across a 32x32 tile. +pub(crate) const M5_MAX_TILE_BUFFER: u64 = 65536; const PROCESS_EMPTY: u32 = uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_PROCESS_EMPTY_TILES; const RSRC_SPEC_HI: u32 = uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_RSRC_SPEC_HI; const DBIAS_INT: u32 = uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_DBIAS_IS_INT; const FEW_PRIMITIVES: u32 = uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_FEW_PRIMITIVES; const FRAGMENT_BARRIERS: u32 = uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_FRAGMENT_BARRIERS; +const STAGE_PREFIXES: u32 = crate::file::DRM_ASAHI_RENDER_STAGE_PREFIXES; +const WHOLE_ATTACHMENTS: u32 = crate::file::DRM_ASAHI_RENDER_WHOLE_ATTACHMENTS; +const SHADER_WRITES: u32 = crate::file::DRM_ASAHI_RENDER_SHADER_WRITES; +/// Render flags only the M5 runtime implements. +const M5_RENDER_FLAGS: u32 = SHADER_WRITES | STAGE_PREFIXES | WHOLE_ATTACHMENTS; const RENDER_FLAGS: u32 = - PROCESS_EMPTY | RSRC_SPEC_HI | DBIAS_INT | FEW_PRIMITIVES | FRAGMENT_BARRIERS; + PROCESS_EMPTY | RSRC_SPEC_HI | DBIAS_INT | FEW_PRIMITIVES | FRAGMENT_BARRIERS | SHADER_WRITES + | STAGE_PREFIXES | WHOLE_ATTACHMENTS; /// Attachment settings persist until replaced, including by an empty list. #[derive(Clone, Copy)] @@ -82,13 +91,85 @@ impl Attachments { } Ok(()) } + + /// An M5 render's lists: the firmware never sees them (the pass's tile stores are encoded from + /// its surfaces), so each range is only the host's ordering hint, its footprint + /// (`Vm::render_footprint`). A range may then cover several mappings and the gaps between them: + /// a coarser footprint only orders more. It must still be a range. + fn validate_hint(&self) -> Result { + for range in self.as_slice() { + if range.size == 0 || range.va.checked_add(range.size).is_none() { + return Err(EINVAL); + } + } + Ok(()) + } + + fn validate_as(&self, space: &impl AddressSpace, rule: AttachmentRule) -> Result { + match rule { + AttachmentRule::Mapped => self.validate(space), + AttachmentRule::Hint => self.validate_hint(), + } + } +} + +/// What a render's attachment lists must prove. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum AttachmentRule { + /// Every range is mapped writable: the firmware reads the lists. + Mapped, + /// Every range is a non-empty range that does not wrap: the lists only order renders (M5). + Hint, +} + +/// Where a render command's stage-prefix fields start: after the complete render command. +const RENDER_STAGE_PREFIXES: usize = core::mem::offset_of!(uapi::drm_asahi_cmd_render_m5, stage_prefixes); +// The wire form M5 Mesa sends: the 448-byte command, then 32 bytes of prefixes. +const _: () = { + assert!(RENDER_STAGE_PREFIXES == size_of::()); + assert!(size_of::() == 448); + assert!(size_of::() == 480); +}; + +// SAFETY: Every field is an integer or a `repr(C)` UAPI struct of integers. +unsafe impl AnyBitPattern for uapi::drm_asahi_cmd_render_m5 {} + +/// A render command. With DRM_ASAHI_RENDER_STAGE_PREFIXES it is the complete +/// struct drm_asahi_cmd_render_m5, with its reserved fields zero; without it the +/// stage prefixes are absent or zero. +fn render_payload( + bytes: &[u8], +) -> Result<(uapi::drm_asahi_cmd_render, Option)> { + let full: uapi::drm_asahi_cmd_render_m5 = payload(bytes)?; + let (cmd, prefixes) = (full.legacy, full.stage_prefixes); + let staged = cmd.flags & STAGE_PREFIXES != 0; + let valid = if staged { + bytes.len() == size_of::() + && cmd.flags & FRAGMENT_BARRIERS == 0 + && prefixes.reserved0 == 0 + && prefixes.reserved1 == 0 + && prefixes.reserved2 == 0 + && prefixes.reserved3 == 0 + } else { + !bytes.get(RENDER_STAGE_PREFIXES..).is_some_and(|tail| tail.iter().any(|&b| b != 0)) + }; + if !valid { + return Err(EINVAL); + } + Ok((cmd, staged.then_some(prefixes))) } // Commands stay inline in the submission plan, avoiding an allocation per render pass. #[allow(clippy::large_enum_variant)] pub(crate) enum Payload { - Render(uapi::drm_asahi_cmd_render, [Attachments; 2], [RayQueryStage; 2]), - Compute(uapi::drm_asahi_cmd_compute, Attachments, IntersectionTables), + Render( + uapi::drm_asahi_cmd_render, + Option, + [Attachments; 2], + [RayQueryStage; 2], + Option>, + ), + Compute(uapi::drm_asahi_cmd_compute, Attachments, IntersectionTables, Option>), } pub(crate) struct Command { @@ -103,6 +184,8 @@ pub(crate) struct Parser<'a> { bytes: &'a [u8], counts: [u16; 2], attachments: [Attachments; 3], + render_resources: Option>, + compute_resources: Option>, } impl<'a> Parser<'a> { @@ -111,6 +194,8 @@ impl<'a> Parser<'a> { bytes, counts: [0; 2], attachments: [Attachments::EMPTY; 3], + render_resources: None, + compute_resources: None, } } @@ -138,18 +223,26 @@ impl<'a> Parser<'a> { { self.counts[0] += 1; no_timestamp_copies(bytes, RENDER_TIMESTAMP_COPIES)?; + // The ray-query stages follow prior_cdm, and the stage-prefix fields follow them. + let ray_query = bytes + .get(RENDER_BASE_SIZE..bytes.len().min(RENDER_STAGE_PREFIXES)) + .unwrap_or(&[]); + let (cmd, stage_prefixes) = render_payload(bytes)?; Payload::Render( - payload(&bytes[..bytes.len().min(RENDER_BASE_SIZE)])?, + cmd, + stage_prefixes, [self.attachments[0], self.attachments[1]], - RayQueryStage::parse_pair(bytes.get(RENDER_BASE_SIZE..).unwrap_or(&[]))?, + RayQueryStage::parse_pair(ray_query)?, + self.render_resources.clone(), ) } else { self.counts[1] += 1; no_timestamp_copies(bytes, COMPUTE_TIMESTAMP_COPIES)?; Payload::Compute( - payload(&bytes[..bytes.len().min(COMPUTE_BASE_SIZE)])?, + payload(bytes)?, self.attachments[2], IntersectionTables::parse(bytes.get(COMPUTE_BASE_SIZE..).unwrap_or(&[]))?, + self.compute_resources.clone(), ) }; return Ok(Some(Command { barriers, payload })); @@ -164,6 +257,25 @@ impl<'a> Parser<'a> { - uapi::drm_asahi_cmd_type_DRM_ASAHI_SET_VERTEX_ATTACHMENTS as usize; self.attachments[slot] = Attachments::parse(bytes)?; } + uapi::drm_asahi_cmd_type_DRM_ASAHI_SET_RENDER_RESOURCES => { + if barriers != [uapi::DRM_ASAHI_BARRIER_NONE as u16; 2] { + return Err(EINVAL); + } + self.render_resources = if bytes.is_empty() { None } + else { Some(Arc::new(Programs::parse(bytes)?, GFP_KERNEL)?) }; + } + uapi::drm_asahi_cmd_type_DRM_ASAHI_SET_RENDER_RESOURCES_V2 => { + if barriers != [uapi::DRM_ASAHI_BARRIER_NONE as u16; 2] { return Err(EINVAL); } + self.render_resources = if bytes.is_empty() { None } + else { Some(Arc::new(Programs::parse_v2(bytes)?, GFP_KERNEL)?) }; + } + uapi::drm_asahi_cmd_type_DRM_ASAHI_SET_COMPUTE_RESOURCES => { + if barriers != [uapi::DRM_ASAHI_BARRIER_NONE as u16; 2] { + return Err(EINVAL); + } + self.compute_resources = if bytes.is_empty() { None } + else { Some(Arc::new(ComputePrograms::parse(bytes)?, GFP_KERNEL)?) }; + } _ => return Err(EINVAL), } } @@ -247,7 +359,13 @@ impl UscWindow { } } -fn sampler(space: &impl AddressSpace, heap: u64, count: u32) -> Result { +/// A command's sampler heap. The M5 command set has none: its render and compute +/// programs carry no sampler-heap state, so M5 refuses a heap here, before anything is +/// queued, rather than run the command without it. +fn sampler(space: &impl AddressSpace, heap: u64, count: u32, m5: bool) -> Result { + if m5 && (heap != 0 || count != 0) { + return Err(EOPNOTSUPP); + } if count > MAX_SAMPLERS || (heap == 0) != (count == 0) || heap & (SAMPLER_SIZE - 1) != 0 { return Err(EINVAL); } @@ -311,13 +429,29 @@ pub(crate) enum Validated { long_job: bool, pass: RenderPass, ray_query: [RayQueryStage; 2], + resources: Option>, + plan: Option, attachments: [Attachments; 2], timestamps: [uapi::drm_asahi_timestamps; 2], }, Compute { long_job: bool, intersection: IntersectionTables, + matrix_ops: bool, + /// The command may copy or reset queries, so it must follow the host's + /// timestamp publication for earlier commands. + query_access: bool, + /// The command's writes are visible when it completes (M5). + visible_completion: bool, + resources: Option>, + plan: Option, scratch: ScratchRequest, + cdm_auxiliary_allocation: u32, + /// The M5 intersection-table binding (dj's `intersection`; that name is the G17P + /// tables' above). + m5_intersection: super::fw::m5::compute::IntersectionTables, + /// The M5 shader-context scratch request (`scratch` is G17P's). + m5_scratch: super::fw::m5::compute::ScratchRequest, usc_base: u64, cdm_va: u64, cdm_end_va: u64, @@ -343,8 +477,20 @@ impl Validated { impl Payload { pub(crate) fn prior(&self) -> [u64; 2] { match self { - Self::Render(cmd, _, _) => [cmd.prior_vdm, cmd.prior_cdm], - Self::Compute(cmd, _, _) => [cmd.prior_vdm, cmd.prior_cdm], + Self::Render(cmd, ..) => [cmd.prior_vdm, cmd.prior_cdm], + Self::Compute(cmd, ..) => [cmd.prior_vdm, cmd.prior_cdm], + } + } + + /// The tiling half's own render and compute prefixes (barrier, exact prior), if the + /// command carries them. + pub(crate) fn tiling_prefixes(&self) -> Option<([u16; 2], [u64; 2])> { + match self { + Self::Render(_, Some(p), ..) => Some(( + [p.ta_vdm_barrier, p.ta_cdm_barrier], + [p.ta_prior_vdm, p.ta_prior_cdm], + )), + _ => None, } } @@ -354,9 +500,14 @@ impl Payload { variant: &super::fw::Variant, space: &impl AddressSpace, ) -> Result { + // Everything M5 accepts beyond the G17P command set is gated on this. + let m5 = variant.m5_commands; + let max_tile_buffer = if m5 { M5_MAX_TILE_BUFFER } else { MAX_TILE_BUFFER }; + let attachment_rule = if m5 { AttachmentRule::Hint } else { AttachmentRule::Mapped }; match self { - Self::Render(cmd, attachments, ray_query) => { - if cmd.flags & !RENDER_FLAGS != 0 + Self::Render(cmd, _, attachments, ray_query, resources) => { + if cmd.flags & !(if m5 { RENDER_FLAGS } else { RENDER_FLAGS & !M5_RENDER_FLAGS }) != 0 + || (!m5 && resources.is_some()) || !(1..=MAX_DIMENSION).contains(&cmd.width_px) || !(1..=MAX_DIMENSION).contains(&cmd.height_px) || !(1..=MAX_LAYERS).contains(&cmd.layers) @@ -371,13 +522,14 @@ impl Payload { 1 => 0, 2 => 1, 4 => 2, + 8 if m5 => 3, _ => return Err(EINVAL), }; if u64::from(cmd.sample_size_B) * u64::from(cmd.utile_width_px) * u64::from(cmd.utile_height_px) * u64::from(cmd.samples) - > MAX_TILE_BUFFER + > max_tile_buffer || cmd.vdm_ctrl_stream_base == 0 || cmd.vdm_ctrl_stream_base & 3 != 0 || cmd.isp_scissor_base == 0 @@ -400,7 +552,7 @@ impl Payload { } let depth = surface(space, &cmd.depth, cmd.layers)?; let stencil = surface(space, &cmd.stencil, cmd.layers)?; - sampler(space, cmd.sampler_heap, cmd.sampler_count.into())?; + sampler(space, cmd.sampler_heap, cmd.sampler_count.into(), m5)?; if !helper_is_empty(&cmd.vertex_helper) || !helper_is_empty(&cmd.fragment_helper) { return Err(EINVAL); } @@ -408,7 +560,7 @@ impl Payload { window.program(space, program.usc)?; } for list in attachments { - list.validate(space)?; + list.validate_as(space, attachment_rule)?; } let pass = RenderPass { width: cmd.width_px.into(), @@ -427,6 +579,9 @@ impl Payload { depth_bias_is_int: cmd.flags & DBIAS_INT != 0, few_primitives: cmd.flags & FEW_PRIMITIVES != 0 && variant.few_primitives, fragment_barriers: cmd.flags & FRAGMENT_BARRIERS != 0, + stage_prefixes: cmd.flags & STAGE_PREFIXES != 0, + whole_attachments: cmd.flags & WHOLE_ATTACHMENTS != 0, + shader_writes_memory: cmd.flags & SHADER_WRITES != 0, occlusion_query_va: cmd.isp_oclqry_base, sampler_heap_va: cmd.sampler_heap, sampler_count: cmd.sampler_count.into(), @@ -459,7 +614,7 @@ impl Payload { }; for stage in ray_query { if (stage.intersection.enabled() || stage.scratch.enabled()) - && !variant.ray_traversal + && !(variant.ray_traversal && variant.render_ray_query) { return Err(refuse_ray_traversal()); } @@ -469,18 +624,41 @@ impl Payload { if code != 0 { require_range(space, code, 64, Access::Read)?; } } } + // M5's per-stage intersection tables, from the render's programs. + let mut m5_tables = [false; 2]; + if let Some(programs) = resources { + for (index, stage) in programs.stage_scs()?.into_iter().enumerate() { + let intersection = stage.intersection; + if intersection.enabled() { + require_range(space, intersection.pool, intersection.pool_bytes, Access::Read)?; + let code = intersection.default_program & 0x0000_ffff_ffff_ffff; + if code != 0 { require_range(space, code, 64, Access::Read)?; } + m5_tables[index] = true; + } + } + } Ok(Validated::Render { long_job: super::timeout::select(false, - ray_query.map(|stage| stage.intersection.enabled()), false), + [0, 1].map(|i| ray_query[i].intersection.enabled() || m5_tables[i]), false), pass, ray_query: *ray_query, + resources: resources.clone(), + plan: None, attachments: *attachments, timestamps: [cmd.ts_vtx, cmd.ts_frag], }) } - Self::Compute(cmd, attachments, intersection) => { - let as_build = super::timeout::as_build(cmd.flags).ok_or(EINVAL)?; - if cmd.cdm_ctrl_stream_base & 3 != 0 + Self::Compute(cmd, attachments, intersection, resources) => { + let as_build = if m5 { false } else { super::timeout::as_build(cmd.flags).ok_or(EINVAL)? }; + if (m5 + && (cmd.flags + & !(crate::file::DRM_ASAHI_COMPUTE_USES_MATRIX + | crate::file::DRM_ASAHI_COMPUTE_NO_QUERY_ACCESS + | crate::file::DRM_ASAHI_COMPUTE_VISIBLE_COMPLETION) + != 0 + || cmd.reserved != 0)) + || (!m5 && resources.is_some()) + || cmd.cdm_ctrl_stream_base & 3 != 0 || cmd.cdm_ctrl_stream_end & 3 != 0 || cmd.cdm_ctrl_stream_end <= cmd.cdm_ctrl_stream_base || !helper_is_empty(&cmd.helper) @@ -493,21 +671,55 @@ impl Payload { cmd.cdm_ctrl_stream_end - cmd.cdm_ctrl_stream_base, Access::Read, )?; - sampler(space, cmd.sampler_heap, cmd.sampler_count)?; + sampler(space, cmd.sampler_heap, cmd.sampler_count, m5)?; attachments.validate(space)?; - let scratch = ScratchRequest::new(cmd.scs_layout)?; + // M5 shader-context scratch has its own (Gen2) layout; G17P's stays disabled. + let scratch = ScratchRequest::new(if m5 { 0 } else { cmd.scs_layout })?; + let m5_scratch = if m5 { + super::fw::m5::compute::ScratchRequest::new(cmd.scs_layout)? + } else { + super::fw::m5::compute::ScratchRequest::NONE + }; if (scratch.enabled() || intersection.enabled()) && !variant.ray_traversal { return Err(refuse_ray_traversal()); } + // The auxiliary allocation is the per-USC page count of the dispatch's SCS + // storage, which the pool is sized for (`plan_compute_scs`). Without SCS there + // is no such storage, as render's programs require. + if !m5_scratch.enabled() && cmd.cdm_auxiliary_allocation != 0 { + return Err(EINVAL); + } if intersection.enabled() { require_range(space, intersection.pool, intersection.pool_bytes, Access::Read)?; let code = intersection.default_program & 0x0000_ffff_ffff_ffff; if code != 0 { require_range(space, code, 64, Access::Read)?; } } + let m5_intersection = if m5 { + super::fw::m5::compute::IntersectionTables::new( + cmd.intersection_pool, cmd.intersection_pool_bytes, + cmd.intersection_first, cmd.intersection_count, + cmd.intersection_default_program, cmd.intersection_default_control)? + } else { + super::fw::m5::compute::IntersectionTables::NONE + }; + if m5_intersection.enabled() { + require_range(space, m5_intersection.pool, m5_intersection.pool_bytes, Access::Read)?; + let code = m5_intersection.default_program & 0x0000_ffff_ffff_ffff; + if code != 0 { require_range(space, code, 64, Access::Read)?; } + } Ok(Validated::Compute { - long_job: super::timeout::select(intersection.enabled(), [false; 2], as_build), + long_job: super::timeout::select(intersection.enabled() || m5_intersection.enabled(), + [false; 2], as_build), intersection: *intersection, + matrix_ops: cmd.flags & crate::file::DRM_ASAHI_COMPUTE_USES_MATRIX != 0, + query_access: cmd.flags & crate::file::DRM_ASAHI_COMPUTE_NO_QUERY_ACCESS == 0, + visible_completion: m5 + && cmd.flags & crate::file::DRM_ASAHI_COMPUTE_VISIBLE_COMPLETION != 0, + resources: resources.clone(), plan: None, scratch, + m5_scratch, + cdm_auxiliary_allocation: cmd.cdm_auxiliary_allocation, + m5_intersection, usc_base: window.base, cdm_va: cmd.cdm_ctrl_stream_base, cdm_end_va: cmd.cdm_ctrl_stream_end - 4, @@ -525,3 +737,42 @@ fn refuse_ray_traversal() -> Error { kernel::pr_warn_once!("G17: ray-traversal work refused on this firmware\n"); Error::from_errno(-(kernel::bindings::EOPNOTSUPP as i32)) } + +#[kunit_tests(asahi_g17_attachments)] +mod attachment_tests { + use super::*; + + /// A VM with nothing mapped. + struct Unmapped; + + impl AddressSpace for Unmapped { + fn covers(&self, _address: u64, _size: u64, _access: Access) -> bool { + false + } + } + + fn list(ranges: &[(u64, u64)]) -> Attachments { + let mut list = Attachments::EMPTY; + for (slot, &(va, size)) in list.ranges.iter_mut().zip(ranges) { + *slot = WriteRange { va, size }; + list.count += 1; + } + list + } + + #[test] + fn an_m5_hint_may_span_unmapped_gaps() { + // One range over two attachments and the gap between them, as userspace merges more + // attachments than a list holds. + let merged = list(&[(0x1_0000_0000, 0x40_0000), (0x2_0000_0000, 0x1_0000_0000)]); + assert_eq!(merged.validate_as(&Unmapped, AttachmentRule::Hint), Ok(())); + assert_eq!(merged.validate_as(&Unmapped, AttachmentRule::Mapped), Err(EINVAL)); + } + + #[test] + fn an_m5_hint_is_still_a_range() { + assert_eq!(list(&[(0x1_0000_0000, 0)]).validate_hint(), Err(EINVAL)); + assert_eq!(list(&[(u64::MAX - 0xfff, 0x2000)]).validate_hint(), Err(EINVAL)); + assert_eq!(list(&[]).validate_hint(), Ok(())); + } +} diff --git a/drivers/gpu/drm/asahi/g17/completion.rs b/drivers/gpu/drm/asahi/g17/completion.rs index 2df1b9cdc34b8..4f04cf4f89972 100644 --- a/drivers/gpu/drm/asahi/g17/completion.rs +++ b/drivers/gpu/drm/asahi/g17/completion.rs @@ -341,6 +341,31 @@ impl Drop for PooledTimestamps { /// Republications of one command whose earlier publications firmware dropped unexecuted. pub(crate) const MAX_REPLAYS: u8 = 3; +/// M5 timestamp pairs share a retained page of the physical compute queue. +/// A completion keeps its backing alive through quarantine and replay. +struct M5ComputeTimestamps { + object: Arc, + offset: usize, +} + +impl M5ComputeTimestamps { + fn clear(&self) -> Result { + let start = self.object.dword(self.offset)?; + let end = self.object.dword(self.offset + 8)?; + start.store(0, Ordering::Relaxed); + end.store(0, Ordering::Relaxed); + barrier(Ordering::SeqCst); + Ok(()) + } + + fn words(&self) -> Result<[u64; 2]> { + let words = [self.object.dword(self.offset)?.load(Ordering::Relaxed), + self.object.dword(self.offset + 8)?.load(Ordering::Relaxed)]; + barrier(Ordering::Acquire); + Ok(words) + } +} + #[pin_data] pub(crate) struct Completion { status: Arc, @@ -349,6 +374,11 @@ pub(crate) struct Completion { // predecessor's host writes. DRM credits must follow the terminal result. terminal: Option>, compute: Option, + #[pin] + m5_compute: Mutex>, + /// A compute command. M5 compute binds its timestamps at publication, so + /// `compute` alone does not identify the engine. + is_compute: bool, feed: Option>, usage: Arc, sources: Option>, @@ -407,7 +437,9 @@ impl Completion { } else { HostOutput::Immediate(output) }; - let compute = if compute { + // M5 compute binds its timestamp pair in its physical queue's page at publication. + let is_compute = compute; + let compute = if compute && !context.is_m5() { Some(context.compute_timestamps().take(dev, context.vm())?) } else { None @@ -418,6 +450,8 @@ impl Completion { output, terminal, compute, + m5_compute <- new_mutex!(None, "M5 queue timestamps"), + is_compute, feed, usage, sources, @@ -521,14 +555,33 @@ impl Completion { } pub(crate) fn compute_timestamp_va(&self) -> Result { - Ok(self.compute.as_ref().ok_or(EINVAL)?.page()?.mapping.iova()) + if let Some(compute) = &self.compute { return Ok(compute.page()?.mapping.iova()); } + let compute = self.m5_compute.lock(); + let compute = compute.as_ref().ok_or(EINVAL)?; + Ok(compute.object.gpu_va() + compute.offset as u64) + } + + /// The queue checked its record-slot retirement before calling this. + /// Clearing and binding precede descriptor publication under the device lock. + pub(crate) fn bind_m5_compute_timestamps(&self, object: &Arc, offset: usize) -> Result { + if self.compute.is_some() || self.signalled.load(Ordering::Acquire) { + return Err(EINVAL); + } + let timestamps = M5ComputeTimestamps { object: object.clone(), offset }; + timestamps.clear()?; + let address = timestamps.object.gpu_va() + offset as u64; + *self.m5_compute.lock() = Some(timestamps); + Ok(address) } /// The caller has proved that firmware retired this command, or has /// received its scheduler kill acknowledgement with recovery closed. /// These are the command's public timestamp words, not queue-private state. pub(crate) fn compute_words(&self) -> Result<[u64; 2]> { - let object = &self.compute.as_ref().ok_or(EINVAL)?.page()?.object; + let Some(compute) = self.compute.as_ref() else { + return self.m5_compute.lock().as_ref().ok_or(EINVAL)?.words(); + }; + let object = &compute.page()?.object; let words = [ object.dword(0)?.load(Ordering::Relaxed), object.dword(8)?.load(Ordering::Relaxed), @@ -581,7 +634,10 @@ impl Completion { if self.signalled.load(Ordering::Acquire) { return Err(ECANCELED); } - let object = &self.compute.as_ref().ok_or(EINVAL)?.page()?.object; + let Some(compute) = self.compute.as_ref() else { + return self.m5_compute.lock().as_ref().ok_or(EINVAL)?.clear(); + }; + let object = &compute.page()?.object; let start = object.dword(0)?; let end = object.dword(8)?; start.store(0, Ordering::Relaxed); @@ -760,7 +816,21 @@ impl Completion { /// Exact retirement permits the physical lease and VM pin to be released /// before waking userspace. A failure fence alone never establishes this. #[track_caller] - pub(crate) fn complete(&self, mut result: Result<[u64; 4]>) { + pub(crate) fn complete(&self, result: Result<[u64; 4]>) { + // M5 compute feeds the refill detector only from its exact execution record + // (RetiredCompute); M5 compute is the compute without pooled timestamps. + let execution = if self.is_compute && self.compute.is_none() { + None + } else { + result.as_ref().ok().map(|ticks| [ticks[0], ticks[1]]) + }; + self.complete_with_compute(result, execution); + } + + /// Compute feedback is separate from public timestamps, which may contain + /// a synthesized retirement envelope when the execution record is absent. + #[track_caller] + fn complete_with_compute(&self, mut result: Result<[u64; 4]>, compute: Option<[u64; 2]>) { if result.is_ok() { if let Err(error) = self.check_source_inputs() { result = Err(error); @@ -775,7 +845,7 @@ impl Completion { if let Ok(ticks) = &result { use crate::usage::Engine; let hz = cfg::COMMAND_TIMESTAMP_HZ as u32; - if self.compute.is_some() { + if self.is_compute { self.usage.add(Engine::Compute, ticks[0], ticks[1], hz); } else { self.usage.add(Engine::Vertex, ticks[0], ticks[1], hz); @@ -785,8 +855,10 @@ impl Completion { if let (Some(feed), Ok(ticks)) = (&self.feed, &result) { // Compute reports its dispatch envelope in the first pair, a // render its fragment pass in the second. - if self.compute.is_some() { - feed.note_pass(ticks[0], ticks[1]); + if self.is_compute { + if let Some([start, end]) = compute { + feed.note_pass(start, end); + } } else { feed.note_pass(ticks[2], ticks[3]); } @@ -849,6 +921,8 @@ fn discarded(error: Error) -> bool { /// or `Release`; `FailedOwned` leaves their physical and VM ownership intact. pub(crate) enum Deferred { Retired(Arc, Result<[u64; 4]>), + /// Only the optional interval comes from an exact compute execution record. + RetiredCompute(Arc, [u64; 4], Option<[u64; 2]>), Drained(Arc, Result<[u64; 4]>), FailedOwned(Arc, Error), Unpublished(Arc, Error), @@ -858,6 +932,10 @@ pub(crate) enum Deferred { impl Deferred { pub(crate) fn finish(self) { match self { + Self::RetiredCompute(completion, ticks, execution) => { + completion.complete_with_compute(Ok(ticks), execution); + completion.release_after_retirement(); + } Self::Retired(completion, result) => { completion.complete(result); // An earlier failure may already have signalled this fence while diff --git a/drivers/gpu/drm/asahi/g17/context.rs b/drivers/gpu/drm/asahi/g17/context.rs index 87572f7eb102f..1f953697dd6b7 100644 --- a/drivers/gpu/drm/asahi/g17/context.rs +++ b/drivers/gpu/drm/asahi/g17/context.rs @@ -16,6 +16,10 @@ use super::{ use crate::mmu::{self, ExecutionContext, Vm, UAT_PGSZ}; use core::sync::atomic::{fence, AtomicBool, AtomicU32, AtomicU8, AtomicUsize, Ordering}; use identity::{Order, RenderSlots, WorkStates, FREE, NODE_STRIDE, STATE_COUNT}; + +/// Both of a context's render channels are bound to other pairs whose publications have not +/// all retired: the publisher waits for that retirement. +pub(crate) const RENDER_CHANNEL_PENDING: Error = ERECALLCONFLICT; use super::job::Engine; use kernel::{ new_mutex, @@ -27,6 +31,8 @@ const QOS_IDS: usize = 128; struct QosState { used: u128, + /// Released IDs whose firmware QoS rows still need clearing. + released: u128, /// When each ID was last returned; reuse takes the one returned longest ago. returns: [u64; QOS_IDS], /// Firmware progress a returned ID waits for (see `channel::control_fence`). @@ -47,7 +53,7 @@ impl QosIds { Arc::pin_init( pin_init!(Self { state <- new_mutex!( - QosState { used: 3, returns: [0; QOS_IDS], fences: [0; QOS_IDS], clock: 0 }, + QosState { used: 3, released: 0, returns: [0; QOS_IDS], fences: [0; QOS_IDS], clock: 0 }, "G17 QoS IDs" ) }), @@ -65,6 +71,8 @@ impl QosIds { return Err(super::capacity::refused(super::capacity::Site::QosIds)); }; state.used |= 1u128 << id; + // A new owner writes its own scheduler row at its first publication. + state.released &= !(1u128 << id); id }; // Drop returns the ID if the allocation fails. @@ -98,6 +106,7 @@ impl Drop for QosLease { state.returns[id] = state.clock; state.fences[id] = fence; state.used &= !(1u128 << id); + state.released |= 1u128 << id; } } @@ -157,10 +166,35 @@ impl WorkStorage { } } +/// A context's scheduler page. The M5 firmware caches its mapping, so on M5 it +/// comes from the retention pool and stays mapped until firmware stop; elsewhere +/// it is an ordinary allocation. +pub(crate) enum SchedulerStorage { + Ordinary(KernelObject), + Retained(PooledObject), +} +impl core::ops::Deref for SchedulerStorage { + type Target = KernelObject; + fn deref(&self) -> &KernelObject { + match self { + Self::Ordinary(object) => object, + Self::Retained(object) => object, + } + } +} +impl core::ops::DerefMut for SchedulerStorage { + fn deref_mut(&mut self) -> &mut KernelObject { + match self { + Self::Ordinary(object) => object, + Self::Retained(object) => object, + } + } +} + /// Firmware-visible pages of a context, retained by an installed queue whose /// last owner has exited (see [`Context::retain_firmware_pages`]). pub(crate) struct FirmwarePages { - _scheduler: Arc, + _scheduler: Arc, _work: Arc>, /// The queue record still names the context's execution identity and QoS ID. _hold: Arc, @@ -168,11 +202,13 @@ pub(crate) struct FirmwarePages { } pub(crate) struct Context { + /// The GPU runs the M5 firmware's protocol. + m5: bool, execution: Arc, /// This context's own hold on its execution identity, cloned by retained queue records. hold: Arc, status: Arc, - scheduler: Arc, + scheduler: Arc, scheduler_generation: u32, published: AtomicBool, /// A kill of this context was published, through a compute queue or on its render pairs' @@ -221,12 +257,30 @@ impl Context { let status = execution.vm().status().ok_or(EINVAL)?.clone(); let hold = execution.hold(super::channel::control_fence)?; let qos = qos.acquire()?; - let mut scheduler = alloc.kernel( - UAT_PGSZ, - UAT_PGSZ as u64, - mmu::PROT_GPU_FW_SHARED_RW, - CpuMap::WriteCombined, - )?; + // On M5 only the firmware reads and writes the scheduler and work-state pages; the GPU + // has no access to them. + let kernel_prot = if alloc.uat.is_m5() { + mmu::PROT_FW_SHARED_RW + } else { + mmu::PROT_GPU_FW_SHARED_RW + }; + let mut scheduler = if alloc.uat.is_m5() { + SchedulerStorage::Retained(pool.allocate( + alloc, + Family::Scheduler, + UAT_PGSZ, + UAT_PGSZ, + kernel_prot, + CpuMap::WriteCombined, + )?) + } else { + SchedulerStorage::Ordinary(alloc.kernel( + UAT_PGSZ, + UAT_PGSZ as u64, + mmu::PROT_GPU_FW_SHARED_RW, + CpuMap::WriteCombined, + )?) + }; scheduler.initialize::(0, |value| { value.init(); Ok(()) @@ -239,7 +293,7 @@ impl Context { Family::WorkState, 2 * UAT_PGSZ, UAT_PGSZ, - mmu::PROT_GPU_FW_SHARED_RW, + kernel_prot, CpuMap::WriteCombined, )?; let work_base = object.gpu_va(); @@ -269,6 +323,7 @@ impl Context { let order = Arc::pin_init(new_mutex!(Order::new(), "G17 context admission order"), GFP_KERNEL)?; Ok(Self { + m5: alloc.uat.is_m5(), execution, hold, status, @@ -327,6 +382,24 @@ impl Context { pub(crate) fn is_current(&self) -> bool { self.execution.is_current() } + /// Claim the context's hardware identity for one published M5 command until it retires. + pub(crate) fn pin(&self) -> Result { + self.execution.pin() + } + pub(crate) fn unpin(&self, pin: mmu::ContextPin) { + self.execution.unpin(pin) + } + /// Claim the queue's QoS slot for one published M5 command. Stage 1: the slot is the + /// context's own, taken at creation. + pub(crate) fn pin_qos(&self) -> Result { + Ok(self.qos_id()) + } + pub(crate) fn unpin_qos(&self, _slot: u8) {} + + pub(crate) fn is_m5(&self) -> bool { + self.m5 + } + pub(crate) fn owner_pid(&self) -> u32 { self.owner_pid } @@ -341,7 +414,7 @@ impl Context { } /// Installed physical channels retain this page after the logical execution /// context and its VM lease have completed their scheduler release. - pub(crate) fn scheduler_owner(&self) -> Arc { + pub(crate) fn scheduler_owner(&self) -> Arc { self.scheduler.clone() } /// Scheduler release and command retirement have both been witnessed. @@ -442,21 +515,44 @@ impl Context { alloc: &Allocator<'_>, ids: &Arc, render_global: &Vm, + plan: Option, ) -> Result> { if !self.is_current() { return Err(EFAULT); } + if alloc.uat.is_m5() != plan.is_some() { + return Err(EINVAL); + } let mut render = self.render.lock(); if let Some(pool) = &*render { - // A pool vacated under memory pressure is re-armed before reuse. - pool.ensure_backing(alloc, render_global, self.vm())?; - return Ok(pool.clone()); + if pool.fits(plan.as_ref().map(|p| p.geometry)) { + // A pool vacated under memory pressure is re-armed before reuse. + // M5 backing is installed by the pair that pins the pool. + if !alloc.uat.is_m5() { + pool.ensure_backing(alloc, render_global, self.vm())?; + } + return Ok(pool.clone()); + } + // M5: existing pairs retain the old pool through their rebind and + // its release acknowledgement. Allocation failure leaves it installed. } let pool = Arc::new( - RenderPool::new(alloc, ids, render_global, self.vm(), self.render_blocks())?, + match plan { + Some(plan) => RenderPool::m5( + alloc, + super::freelist::RenderId::logical(ids)?, + self.vm(), + plan.geometry, + )?, + None => { + RenderPool::new(alloc, ids, render_global, self.vm(), self.render_blocks())? + } + }, GFP_KERNEL, )?; - *render = Some(pool.clone()); + let previous = render.replace(pool.clone()); + drop(render); + drop(previous); Ok(pool) } /// Drops the pool once its free-list release is consumed and the closed @@ -484,17 +580,20 @@ pub(crate) struct WorkStateLease { } impl WorkStateLease { pub(super) fn status(&self) -> &Arc { self.context.status() } - pub(crate) fn deferred(context: Arc) -> Self { - let ordinal = context.order.lock().assign(); - Self { + /// The lease of a new ioctl, ordered after every earlier one of its context. EBUSY while + /// too many of the context's ioctls are queued behind its oldest waiting one + /// (see `identity::Order`). + pub(crate) fn deferred(context: Arc) -> Result { + let ordinal = context.order.lock().assign().ok_or(EBUSY)?; + Ok(Self { context, word: AtomicU32::new(FREE), engine: AtomicU8::new(0), ordinal, - } + }) } - /// The ioctl's node, taking one for `engine`'s admission if it has none yet. EBUSY: too many - /// older ioctls still wait for theirs (see `identity::Order`), or no node may be taken now + /// The ioctl's node, taking one for `engine`'s admission if it has none yet. EBUSY: an older + /// ioctl still waits for its node (see `identity::Order`), or no node may be taken now /// (see `identity::may_take`); an older ioctl's admission or a return lets it try again. pub(crate) fn acquire(&self, engine: Engine) -> Result { let current = self.word(); @@ -535,6 +634,15 @@ impl WorkStateLease { pub(crate) fn word(&self) -> u32 { self.word.load(Ordering::Acquire) } + /// The oldest node of this ioctl's context still in use ahead of this ioctl's node, whose + /// draining the firmware's flush of this node waits for; none without a node of its own. + pub(crate) fn older_node(&self) -> Option { + let word = self.word(); + if word == FREE { + return None; + } + self.context.states.older_live(word) + } pub(crate) fn node_va(&self) -> Result { let word = self.word(); if word == FREE { @@ -565,9 +673,35 @@ impl WorkStateLease { .change_count(self.word(), kicks, false) } - pub(crate) fn render_slot(&self, pair: u8) -> Result { - self.context.work.lock().render.get(pair).ok_or_else(|| - super::capacity::refused(super::capacity::Site::RenderWorkSlots)) + /// The render channel of `pair` for a publication. When both channels are bound to other + /// pairs, the least recently used one whose every publication `retired(qids, ordinal)` + /// moves to `pair`; otherwise [`RENDER_CHANNEL_PENDING`]: the caller waits for retirement + /// instead of failing the client. + pub(crate) fn render_channel( + &self, + pair: u8, + retired: impl Fn([u8; 2], u64) -> bool, + ) -> Result { + let (index, previous) = self + .context + .work + .lock() + .render + .bind(pair, retired) + .ok_or(RENDER_CHANNEL_PENDING)?; + if let Some(previous) = previous { + static REBINDS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); + let n = REBINDS.fetch_add(1, Ordering::Relaxed) + 1; + if n <= 8 || n.is_power_of_two() { + pr_info!("asahi: render channel {} of context {} moved from pair {} to pair {} (rebind {})\n", + index, self.context.id(), previous, pair, n); + } + } + Ok(index) + } + /// Records a publication through `pair`'s channel once it reached firmware. + pub(crate) fn channel_published(&self, pair: u8, qids: [u8; 2], ordinal: u64) { + self.context.work.lock().render.published(pair, qids, ordinal); } pub(crate) fn channel_payload(&self, pair: u8, ordinal: u64) -> Result { self.context @@ -609,6 +743,24 @@ impl QosIds { pub(crate) fn used(&self) -> u32 { (self.state.lock().used & !3u128).count_ones() } + /// Clears, with `forget`, the firmware QoS rows of the IDs released and not taken again: + /// they still name the released context's scheduler page. The IDs `forget` returns, whose + /// rows still have work in flight (or all, if it fails), stay released for the next call. + pub(crate) fn forget_released(&self, forget: impl FnOnce(u128) -> Result) { + let ids = { + let mut state = self.state.lock(); + let ids = state.released & !state.used; + state.released = 0; + ids + }; + if ids == 0 { + return; + } + let left = forget(ids).unwrap_or(ids); + let mut state = self.state.lock(); + // An ID taken again meanwhile has a new owner, which writes its own row. + state.released |= left & !state.used; + } } /// Memory a context's work list may lead the firmware to stays allocated: a linked node (or @@ -646,6 +798,44 @@ mod capacity_tests { assert_eq!(ids.state.lock().used, 3); } } + + #[test] + fn a_released_qos_row_still_in_use_is_cleared_later() { + let ids = QosIds::new().unwrap(); + let (first, second) = (ids.acquire().unwrap(), ids.acquire().unwrap()); + let both = (1u128 << first.id) | (1u128 << second.id); + // Occupy fresh IDs so the callback must reuse one of the returned IDs. + let mut others = KVec::with_capacity(QOS_IDS - 4, GFP_KERNEL).unwrap(); + for _ in 0..QOS_IDS - 4 { + others.push(ids.acquire().unwrap(), GFP_KERNEL).unwrap(); + } + drop((first, second)); + // Work still runs on both rows: neither is cleared, and both stay released. + ids.forget_released(|released| { + assert_eq!(released, both); + Ok(released) + }); + // A failed clear keeps them too. + ids.forget_released(|released| { + assert_eq!(released, both); + Err(EIO) + }); + // One is taken again meanwhile: its new owner writes its own row. + let mut again = None; + ids.forget_released(|released| { + again = Some(ids.acquire().unwrap()); + Ok(released) + }); + let again = again.unwrap(); + assert_ne!(both & (1u128 << again.id), 0); + let rest = both & !(1u128 << again.id); + ids.forget_released(|released| { + assert_eq!(released, rest); + Ok(0) + }); + ids.forget_released(|_| panic!("nothing left to clear")); + drop(again); + } } impl Context { diff --git a/drivers/gpu/drm/asahi/g17/context/identity.rs b/drivers/gpu/drm/asahi/g17/context/identity.rs index 7586f8474e602..0fe045d6f780c 100644 --- a/drivers/gpu/drm/asahi/g17/context/identity.rs +++ b/drivers/gpu/drm/asahi/g17/context/identity.rs @@ -77,12 +77,10 @@ impl WorkStates { } } -/// Ordinals past the turn remembered for ioctls that took a node or left early. +/// How far past the oldest waiting ioctl a new ioctl may be ordered: the ordinals past the +/// turn remembered for ioctls that left early. pub(super) const ORDER_WINDOW: u64 = 1024; -/// How far past the oldest waiting ioctl an ioctl may take its node. -pub(super) const RUN_AHEAD: u64 = 8; - /// The order in which a context's ioctls take work-state nodes. /// /// Each engine admits a context's ioctls in submission order, but the engines advance @@ -93,20 +91,21 @@ pub(super) const RUN_AHEAD: u64 = 8; /// engines, at most 36 between scheduling and completion, and the same load runs there without /// failure. /// -/// So an ioctl takes its node only while at most [`RUN_AHEAD`] older ioctls of its context have -/// neither taken one nor left without one. Native scheduling happens at commit, on one thread; -/// here each engine admits from its own scheduler, and a strictly ordered turn costs a scheduler -/// round trip whenever the other engine's ioctl is ready first. A short run-ahead avoids that -/// without letting one engine get far ahead of the other. +/// So an ioctl takes its node only at its turn, once every older ioctl of its context has taken +/// one or left without one. The firmware sorts a context's nodes by their words and flushes +/// from the head, and a node's word is drawn when the node is taken +/// ([`WorkStates::allocate`]): an ioctl taking its node ahead of an older one would hold a lower +/// word than that ioctl, whose completion stamp would then wait behind a node submitted after +/// it. Taking nodes strictly in turn keeps the words in submission order. Each engine admits +/// from its own scheduler, so the turn costs a scheduler round trip whenever the other engine's +/// ioctl is ready first. pub(super) struct Order { /// The next ordinal to hand out. next: u64, /// The oldest ordinal that has neither taken a node nor left. turn: u64, - /// Ordinals past the turn that took a node or left, as bits at ordinal % ORDER_WINDOW. + /// Ordinals past the turn that left, as bits at ordinal % ORDER_WINDOW. done: [u64; (ORDER_WINDOW / 64) as usize], - /// Set once an ioctl left too far past the turn to remember; admission is unordered after. - unordered: bool, /// An ioctl was refused since the turn last moved: its publisher waits for the next move. refused: bool, } @@ -116,24 +115,28 @@ impl Order { next: 0, turn: 0, done: [0; (ORDER_WINDOW / 64) as usize], - unordered: false, refused: false, } } - /// The ordinal of a new ioctl, in submission order. - pub(super) fn assign(&mut self) -> u64 { + /// The ordinal of a new ioctl, in submission order. None while [`ORDER_WINDOW`] ioctls of + /// the context are past the oldest one still waiting for its node. + pub(super) fn assign(&mut self) -> Option { + if self.next - self.turn >= ORDER_WINDOW { + return None; + } let ordinal = self.next; self.next += 1; - ordinal + Some(ordinal) } - /// Whether the ioctl `ordinal` may take its node now. A refusal is remembered until the - /// turn next moves, so that move wakes the refused publisher. + /// Whether the ioctl `ordinal` may take its node now: only at its turn. A refusal is + /// remembered until the turn next moves, so that move wakes the refused publisher. pub(super) fn admits(&mut self, ordinal: u64) -> bool { - let admits = self.unordered || ordinal <= self.turn + RUN_AHEAD; + let admits = ordinal == self.turn; self.refused |= !admits; admits } - /// The ioctl `ordinal` took its node. Returns whether a refused ioctl may now be admitted. + /// The ioctl `ordinal` took its node at its turn. Returns whether a refused ioctl may now be + /// admitted. pub(super) fn admit(&mut self, ordinal: u64) -> bool { self.finish(ordinal) } @@ -143,17 +146,14 @@ impl Order { self.finish(ordinal) } fn finish(&mut self, ordinal: u64) -> bool { - if self.unordered || ordinal < self.turn { + if ordinal < self.turn { return false; } if ordinal == self.turn { self.pass(); return core::mem::take(&mut self.refused); } - if ordinal - self.turn >= ORDER_WINDOW { - self.unordered = true; - return core::mem::take(&mut self.refused); - } + // assign() keeps every ordinal within the window past the turn. let bit = ordinal % ORDER_WINDOW; self.done[(bit / 64) as usize] |= 1 << (bit % 64); false @@ -172,19 +172,35 @@ impl Order { } } -/// Two stable logical ordinals, independent of physical pair numbering. +/// Two stable logical ordinals (render channels), independent of physical pair numbering. A +/// context's render work can move between physical pairs (sibling pairs, lending, relocation, +/// QID recycling), so a channel is rebound to a new pair once every publication this context +/// made through it has retired. A context's channel counts its passes up for the whole +/// process, so a rebound channel continues its pass sequence. pub(super) struct RenderSlots { pairs: [Option; 2], origins: [Option; 2], + /// Pass of the channel's last publication on its previous pair. + bases: [u8; 2], + /// QIDs and visibility target (ordinal + 1) of the last publication made through each + /// channel. + last: [Option<([u8; 2], u64)>; 2], + /// Recency of each channel's last publication. + used: [u64; 2], + epoch: u64, } impl RenderSlots { pub(super) const fn new() -> Self { Self { pairs: [None; 2], origins: [None; 2], + bases: [0; 2], + last: [None; 2], + used: [0; 2], + epoch: 0, } } - pub(super) fn get(&mut self, pair: u8) -> Option { + fn get(&mut self, pair: u8) -> Option { if let Some(index) = self.pairs.iter().position(|value| *value == Some(pair)) { return Some(index as u32); } @@ -192,14 +208,61 @@ impl RenderSlots { self.pairs[index] = Some(pair); Some(index as u32) } + /// The channel of `pair`, rebinding the least recently used channel whose publications + /// have all retired (`retired(qids, ordinal)`) when both are bound to other pairs. `None`: + /// no channel can move yet. + pub(super) fn bind( + &mut self, + pair: u8, + retired: impl Fn([u8; 2], u64) -> bool, + ) -> Option<(u32, Option)> { + if let Some(index) = self.get(pair) { + return Some((index, None)); + } + let index = (0..2) + .filter(|index| self.last[*index].is_none_or(|(qids, ordinal)| retired(qids, ordinal))) + .min_by_key(|index| self.used[*index])?; + let previous = self.pairs[index]; + // `last` holds the ordinal after the last publication (its visibility target), so that + // publication's pass is base + (next - origin). + if let (Some(origin), Some((_, next))) = (self.origins[index], self.last[index]) { + self.bases[index] = self.bases[index].wrapping_add(next.wrapping_sub(origin) as u8); + } + self.pairs[index] = Some(pair); + self.origins[index] = None; + self.last[index] = None; + Some((index as u32, previous)) + } pub(super) fn payload(&mut self, pair: u8, ordinal: u64) -> Option { - let index = self.get(pair)? as usize; + let index = self.pairs.iter().position(|value| *value == Some(pair))?; let origin = *self.origins[index].get_or_insert(ordinal); - Some(ordinal.checked_sub(origin)?.wrapping_add(1) as u8) + Some(self.bases[index].wrapping_add(ordinal.checked_sub(origin)?.wrapping_add(1) as u8)) + } + /// Records a publication that reached firmware through `pair`'s channel. + pub(super) fn published(&mut self, pair: u8, qids: [u8; 2], ordinal: u64) -> bool { + let Some(index) = self.pairs.iter().position(|value| *value == Some(pair)) else { + return false; + }; + self.epoch = self.epoch.wrapping_add(1); + self.used[index] = self.epoch; + self.last[index] = Some((qids, ordinal)); + true } } impl WorkStates { + /// The oldest node in use that the firmware lists ahead of `word`: a live word older than + /// it by the signed 32-bit difference the firmware sorts by. Its ioctl was submitted + /// earlier ([`Order`]), and until that node drains the firmware withholds every stamp of + /// `word`'s node behind it. + pub(super) fn older_live(&self, word: u32) -> Option { + self.live + .iter() + .map(|live| live.load(Ordering::Acquire)) + .filter(|&live| live != FREE && (live.wrapping_sub(word) as i32) < 0) + .min_by_key(|&live| live.wrapping_sub(word) as i32) + } + pub(super) fn used(&self) -> usize { self.live.iter().filter(|word| word.load(Ordering::Acquire) != FREE).count() } @@ -250,65 +313,66 @@ mod tests { } #[test] - fn ioctls_take_nodes_at_most_run_ahead_past_the_oldest_waiting_one() { + fn ioctls_take_nodes_only_at_their_turn() { let mut order = Order::new(); - let ordinals: [u64; RUN_AHEAD as usize + 3] = core::array::from_fn(|_| order.assign()); - let (oldest, last) = (ordinals[0], ordinals[RUN_AHEAD as usize + 2]); - // Every ioctl up to RUN_AHEAD past the oldest waiting one may take its node. - for &ordinal in &ordinals[..=RUN_AHEAD as usize] { - assert!(order.admits(ordinal)); - } - assert!(!order.admits(ordinals[RUN_AHEAD as usize + 1])); - // Ioctls ahead of the turn taking their nodes do not move it. - for &ordinal in &ordinals[1..=RUN_AHEAD as usize] { - assert!(!order.admit(ordinal)); + let ordinals: [u64; 4] = core::array::from_fn(|_| order.assign().unwrap()); + // Only the oldest waiting ioctl may take its node; the others are refused. + for &ordinal in &ordinals[1..] { + assert!(!order.admits(ordinal)); } - assert!(!order.admits(ordinals[RUN_AHEAD as usize + 1])); - // The oldest one moves the turn past everything already admitted, and wakes the refused. - assert!(order.admit(oldest)); - assert!(order.admits(last)); - let next = order.assign(); - assert!(order.admits(next)); - assert!(!order.admits(next + RUN_AHEAD + 1)); + assert!(order.admits(ordinals[0])); + // Its admission moves the turn and wakes the refused. + assert!(order.admit(ordinals[0])); + assert!(order.admits(ordinals[1])); + // Nobody was refused since, so the next admission wakes nobody. + assert!(!order.admit(ordinals[1])); + assert!(!order.admits(ordinals[3])); + assert!(order.admits(ordinals[2])); + assert!(order.admit(ordinals[2])); + assert!(order.admits(ordinals[3])); } #[test] fn ioctls_leaving_without_a_node_pass_the_turn() { let mut order = Order::new(); - let (a, b, c) = (order.assign(), order.assign(), order.assign()); - let later: [u64; RUN_AHEAD as usize] = core::array::from_fn(|_| order.assign()); - let beyond = later[RUN_AHEAD as usize - 1]; - // b and c fail before their turn; while a waits, ioctls past a + RUN_AHEAD wait too. + let (a, b, c, d) = ( + order.assign().unwrap(), + order.assign().unwrap(), + order.assign().unwrap(), + order.assign().unwrap(), + ); + // b and c fail before their turn; while a waits, d waits too. assert!(!order.leave(b)); assert!(!order.leave(c)); - assert!(!order.admits(beyond)); + assert!(!order.admits(d)); // Once a takes its node, the turn passes b and c as well, and the refused one may retry. assert!(order.admit(a)); - assert!(order.admits(beyond)); - // With nobody refused since, moving the turn wakes nobody; an ordinal already passed - // leaves without effect. - assert!(!order.leave(later[0])); + assert!(order.admits(d)); + // An ordinal already passed leaves without effect. assert!(!order.leave(a)); } #[test] - fn an_ioctl_leaving_too_far_ahead_unorders_admission() { + fn no_ioctl_is_ordered_past_the_window() { let mut order = Order::new(); - let first = order.assign(); + let first = order.assign().unwrap(); let mut last = first; - for _ in 0..ORDER_WINDOW { - last = order.assign(); + for _ in 1..ORDER_WINDOW { + last = order.assign().unwrap(); } - assert!(!order.admits(last)); - assert!(order.leave(last)); - assert!(order.admits(last - 1)); + assert!(order.assign().is_none()); + // An ioctl leaving far ahead is remembered within the window. + assert!(!order.leave(last)); + assert!(order.admits(first)); + assert!(!order.admit(first)); + assert!(order.assign().is_some()); } #[test] fn single_engine_admission_wakes_nobody() { let mut order = Order::new(); - for _ in 0..3 * RUN_AHEAD { - let ordinal = order.assign(); + for _ in 0..24 { + let ordinal = order.assign().unwrap(); assert!(order.admits(ordinal)); assert!(!order.admit(ordinal)); } diff --git a/drivers/gpu/drm/asahi/g17/event.rs b/drivers/gpu/drm/asahi/g17/event.rs index a175e58392379..50df8107bbac9 100644 --- a/drivers/gpu/drm/asahi/g17/event.rs +++ b/drivers/gpu/drm/asahi/g17/event.rs @@ -2,6 +2,9 @@ //! Event service ordering under the device mutex. A restart ends its role's batch; a worker //! joins preparation leases and services it on a separate turn before continuing the rings. +//! On M5 only the event rings are consumed here: the firmware's trace and statistics reports +//! are drained after the completions this service retires have been signalled +//! (`Firmware::drain_reports`), so no report queued ahead of a completion delays its fence. use super::{ fw::channels::{Event, FreeListCompletion, TvbGrowRequest}, @@ -15,6 +18,8 @@ pub(super) trait Transport { fn roles(&self) -> &'static [Role]; fn trace(&mut self, role: Role); fn statistics(&mut self, role: Role); + /// The reports are drained after the pass's completions are signalled (M5). + fn reports_after_signal(&self) -> bool; fn drain( &mut self, role: Role, @@ -91,11 +96,13 @@ pub(super) fn service(host: &mut H) -> Result { return Ok(Pending::default()); } host.observe_render_releases()?; - for &role in host.roles() { - host.trace(role); - } - for &role in host.roles() { - host.statistics(role); + if !host.reports_after_signal() { + for &role in host.roles() { + host.trace(role); + } + for &role in host.roles() { + host.statistics(role); + } } let mut masks = [0; 2]; if !host.recovery().pending() { diff --git a/drivers/gpu/drm/asahi/g17/fans.c b/drivers/gpu/drm/asahi/g17/fans.c new file mode 100644 index 0000000000000..6178a62068608 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fans.c @@ -0,0 +1,24 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * The M5 thermal governor's sensor and fan calls (thermal.rs). The thermal + * core and the SMC hwmon cell have stubs where they are not built; Rust + * reaches them through these. + */ +#include +#include +#include +#include "fans.h" + +int asahi_zone_temp(const char *name, int *temp) +{ + struct thermal_zone_device *zone = thermal_zone_get_zone_by_name(name); + + if (IS_ERR_OR_NULL(zone)) + return zone ? PTR_ERR(zone) : -ENODEV; + return thermal_zone_get_temp(zone, temp); +} + +int asahi_fans_request(struct device *dev, unsigned int permille, unsigned int min_step_rpm) +{ + return macsmc_hwmon_fans_request(dev, permille, min_step_rpm); +} diff --git a/drivers/gpu/drm/asahi/g17/fans.h b/drivers/gpu/drm/asahi/g17/fans.h new file mode 100644 index 0000000000000..4bc18d80d3fe0 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fans.h @@ -0,0 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef _ASAHI_G17_FANS_H +#define _ASAHI_G17_FANS_H +struct device; +int asahi_zone_temp(const char *name, int *temp); +int asahi_fans_request(struct device *dev, unsigned int permille, unsigned int min_step_rpm); +#endif diff --git a/drivers/gpu/drm/asahi/g17/feed.rs b/drivers/gpu/drm/asahi/g17/feed.rs index bd1228a82e26f..a96f67ae6b8f9 100644 --- a/drivers/gpu/drm/asahi/g17/feed.rs +++ b/drivers/gpu/drm/asahi/g17/feed.rs @@ -9,6 +9,9 @@ //! Policy creation and the initial snapshot take the policy rwsem before the //! requests mutex. Removal callbacks and the worker take only requests; //! notifier unregistration joins callbacks without holding that mutex. +//! M5 requests full CPU capacity only while refill pressure keeps the detector +//! active. Sustained demand can renew its deadline indefinitely; withdrawal +//! occurs on a later worker poll, not at an exact eight-millisecond boundary. use crate::hw::t8140::qos::CLOCK_HZ; use core::{ @@ -27,6 +30,7 @@ use core::ptr; use kernel::{bindings, new_mutex, sync::Mutex, types::Opaque}; const MAX_REFILL_TICKS: u64 = CLOCK_HZ / 500; +const _: () = assert!(CLOCK_HZ == crate::hw::t8140::resources::COMMAND_TIMESTAMP_HZ); const STREAK_ON: u32 = 8; const STREAK_MAX: u32 = 64; const LINGER_NS: u64 = 8_000_000; @@ -76,7 +80,7 @@ impl Detector { } fn note(&self, start: u64, end: u64, now: u64) { - if end <= start { + if start == 0 || end <= start { return; } let previous = self.last_end.fetch_max(end, Ordering::AcqRel); @@ -141,6 +145,8 @@ struct Requests { policies: [*mut bindings::cpufreq_policy; MAX_POLICIES], maxima: [u32; MAX_POLICIES], applied: [u32; MAX_POLICIES], + /// Full CPU capacity for M5 refill pressure; other profiles retain their floor. + floor_percent: u64, } // SAFETY: Each nonnull policy owns a kobject reference. The device requests @@ -150,12 +156,21 @@ unsafe impl Send for Requests {} #[cfg(CONFIG_CPU_FREQ)] impl Requests { - fn new() -> Self { + fn new(m5: bool) -> Self { Self { requests: Opaque::zeroed(), policies: [ptr::null_mut(); MAX_POLICIES], maxima: [0; MAX_POLICIES], applied: [0; MAX_POLICIES], + floor_percent: if m5 { 100 } else { FLOOR_PERCENT }, + } + } + + fn target_floor(maximum: u32, floor_percent: u64, enabled: bool) -> u32 { + if enabled { + (u64::from(maximum) * floor_percent / 100) as u32 + } else { + bindings::FREQ_QOS_MIN_DEFAULT_VALUE as u32 } } @@ -206,11 +221,7 @@ impl Requests { if this.policies[index].is_null() { continue; } - let value = if enabled { - (u64::from(this.maxima[index]) * FLOOR_PERCENT / 100) as u32 - } else { - bindings::FREQ_QOS_MIN_DEFAULT_VALUE - }; + let value = Self::target_floor(this.maxima[index], *this.floor_percent, enabled); if this.applied[index] != value { // SAFETY: This slot is registered, pinned and exclusively locked; // the policy reference keeps its constraints alive through REMOVE. @@ -296,12 +307,14 @@ pub(crate) struct Feed { } impl Feed { - pub(crate) fn new() -> Result> { + pub(crate) fn new(m5: bool) -> Result> { + #[cfg(not(CONFIG_CPU_FREQ))] + let _ = m5; let feed = Arc::pin_init( pin_init!(Self { detector: Detector::new(), #[cfg(CONFIG_CPU_FREQ)] - requests <- new_mutex!(Requests::new(), "G17 CPU feed"), + requests <- new_mutex!(Requests::new(m5), "G17 CPU feed"), #[cfg(CONFIG_CPU_FREQ)] notifier <- pin_init!(PolicyNotifier { block: Opaque::new(bindings::notifier_block { @@ -422,6 +435,8 @@ impl Feed { Monotonic::ktime_get() as u64 } + /// Firmware execution intervals in the device's 24 MHz timestamp domain. + /// Missing, unexecuted and wrapped intervals must not affect the detector. pub(crate) fn note_pass(&self, start: u64, end: u64) { if cfg!(CONFIG_CPU_FREQ) { self.detector.note(start, end, Self::now()); diff --git a/drivers/gpu/drm/asahi/g17/fence.rs b/drivers/gpu/drm/asahi/g17/fence.rs index 0b8819bdbd396..38955eb33271d 100644 --- a/drivers/gpu/drm/asahi/g17/fence.rs +++ b/drivers/gpu/drm/asahi/g17/fence.rs @@ -7,7 +7,7 @@ use super::context::{Context, WorkStateLease}; use crate::file::SyncItem; -use core::sync::atomic::{AtomicBool, AtomicI32, AtomicU8, AtomicUsize, Ordering}; +use core::sync::atomic::{AtomicBool, AtomicI32, AtomicU32, AtomicU64, AtomicU8, AtomicUsize, Ordering}; use kernel::{ bindings, c_str, dma_fence::{Fence, FenceContexts, FenceObject, FenceOps, RawDmaFence, UniqueFence, UserFence}, @@ -63,9 +63,44 @@ pub(crate) fn allocation_failure(contexts: &FenceContexts) -> Result { pub(crate) struct ExportFence { source: Option>, scoped: bool, + /// A waiter enabled signalling on this ioctl output fence before it signalled ([`waiters`]). + waited: AtomicBool, } static EXPORT_OPS: AtomicUsize = AtomicUsize::new(0); +/// Output fences a waiter is waiting on: signalling enabled and not yet signalled. +static WAITERS: AtomicU32 = AtomicU32::new(0); +/// Waits begun on output fences, ever. +static WAITS: AtomicU64 = AtomicU64::new(0); + +/// The output fences a waiter is blocked on now, and the waits begun so far: the work a host +/// thread waits for, which the M5 performance effort follows ([`super::power::Effort`]). +pub(crate) fn waiters() -> (u32, u64) { + (WAITERS.load(Ordering::Acquire), WAITS.load(Ordering::Acquire)) +} + +/// The first waiter on an unsignalled output fence (the fence core calls this once, under the +/// fence's lock). +fn waited(flag: &AtomicBool) { + if !flag.swap(true, Ordering::AcqRel) { + WAITERS.fetch_add(1, Ordering::AcqRel); + WAITS.fetch_add(1, Ordering::AcqRel); + } +} + +/// The fence signalled or went away: a waiter on it no longer waits. +fn unwaited(flag: &AtomicBool) { + if flag.swap(false, Ordering::AcqRel) { + WAITERS.fetch_sub(1, Ordering::AcqRel); + } +} + +impl Drop for ExportFence { + fn drop(&mut self) { + unwaited(&self.waited); + } +} + #[vtable] impl FenceOps for ExportFence { fn get_driver_name<'a>(self: &'a FenceObject) -> &'a CStr { @@ -74,6 +109,10 @@ impl FenceOps for ExportFence { fn get_timeline_name<'a>(self: &'a FenceObject) -> &'a CStr { c_str!("g17-aggregate") } + fn enable_signaling(self: &FenceObject) -> bool { + waited(&self.waited); + true + } } #[inline(never)] @@ -83,6 +122,7 @@ fn export_fallback(contexts: &FenceContexts) -> Result> ExportFence { source: None, scoped: false, + waited: AtomicBool::new(false), }, )?; // SAFETY: Unique, initialized and still private. Only this constructor @@ -159,7 +199,7 @@ pub(crate) struct Submission { impl Submission { fn new(context: Arc, firmware_source: bool) -> Result> { - let work = Arc::new(WorkStateLease::deferred(context), GFP_KERNEL)?; + let work = Arc::new(WorkStateLease::deferred(context)?, GFP_KERNEL)?; Self::with_work(Some(work), firmware_source) } fn with_work(work: Option>, firmware_source: bool) -> Result> { @@ -238,6 +278,7 @@ impl Submission { } else { fence.signal(); } + unwaited(&fence.waited); } } } @@ -510,6 +551,7 @@ impl Drop for Outputs { ); } fallback.signal(); + unwaited(&fallback.waited); } } } diff --git a/drivers/gpu/drm/asahi/g17/freelist.rs b/drivers/gpu/drm/asahi/g17/freelist.rs index 9015dc82e4076..3d1449bee2147 100644 --- a/drivers/gpu/drm/asahi/g17/freelist.rs +++ b/drivers/gpu/drm/asahi/g17/freelist.rs @@ -11,31 +11,206 @@ use super::fw::freelist::*; use super::fw::initdata::{FreeListArgs, FreeListClass, FreeListControl, FREE_LIST_BLOCK_PAGES, FREE_LIST_BLOCK_SIZE}; use super::object::{Allocator, CpuMap, KernelObject}; use crate::mmu; -use core::sync::atomic::{fence, AtomicBool, AtomicU64, Ordering}; +use core::sync::atomic::{fence, AtomicBool, AtomicU16, AtomicU64, Ordering}; use kernel::{ new_mutex, prelude::*, sync::{Arc, Mutex}, }; +mod render_ids; + +const NO_RENDER_ID: u16 = u16::MAX; +static NEXT_RENDER_OWNER: AtomicU64 = AtomicU64::new(1); + +struct RenderIdCell { + owner: u64, + slot: AtomicU16, + generation: AtomicU64, + closed: AtomicBool, + storage: Arc, +} +impl RenderIdCell { + fn has_storage(&self) -> bool { + self.storage.backing.lock().is_some() || self.storage.spare.lock().is_some() + } +} + +/// Detached after the device's final free-list release. Drop outside the +/// device and identity locks because mappings may take memory-management locks. +#[must_use] +pub(crate) struct ReclaimedRenderId { + _backing: Option, + spare: Option, +} +impl ReclaimedRenderId { + /// Caller is outside the device lock. Remove the previous client's alias + /// before handing physical storage to another logical pool. + fn into_pages(self) -> Option { + self._backing.map(RenderBacking::into_pages).or(self.spare) + } +} + +struct RenderIdState { + leases: render_ids::State, + cells: [Option>; render_ids::COUNT], +} + /// Render pools occupy 129..255; compute uses its QID plus one, in 1..128. /// First-free allocation keeps the shared firmware namespace deterministic. +#[pin_data] pub(crate) struct RenderIds { used: [AtomicU64; 2], + available: Arc, + #[pin] + state: Mutex, } impl RenderIds { - pub(crate) const fn new() -> Self { - Self { + #[inline(never)] + pub(crate) fn new() -> Result> { + Arc::pin_init(try_pin_init!(Self { used: [AtomicU64::new(0), AtomicU64::new(1 << 63)], + available: super::availability::Availability::new()?, + state <- new_mutex!(RenderIdState { + leases: render_ids::State::new(), + cells: [const { None }; render_ids::COUNT], + }, "G17 render identities"), + }), GFP_KERNEL) + } + + pub(crate) fn availability(&self) -> &Arc { + &self.available + } + + fn pin(self: &Arc, cell: &Arc, assigned: impl FnOnce()) -> Result { + let mut state = self.state.lock(); + let slot = cell.slot.load(Ordering::Acquire); + if slot != NO_RENDER_ID { + let lease = render_ids::Lease { + slot, owner: cell.owner, generation: cell.generation.load(Ordering::Acquire), + }; + match state.leases.pin(lease) { + Ok(_) => return Ok(RenderIdPin { allocator: self.clone(), cell: cell.clone(), lease }), + Err(render_ids::Error::Overflow) => return Err(EOVERFLOW), + Err(_) => {} + } + } + let claim = state.leases.claim(cell.owner, |_| false).map_err(|error| match error { + render_ids::Error::Busy | render_ids::Error::PendingRelease => ENOSPC, + render_ids::Error::Overflow => EOVERFLOW, + _ => EIO, + })?; + let index = usize::from(claim.lease.slot - render_ids::FIRST); + state.cells[index] = Some(cell.clone()); + assigned(); + cell.generation.store(claim.lease.generation, Ordering::Relaxed); + cell.slot.store(claim.lease.slot, Ordering::Release); + Ok(RenderIdPin { allocator: self.clone(), cell: cell.clone(), lease: claim.lease }) + } + + fn close(&self, cell: &Arc) { + cell.closed.store(true, Ordering::Release); + let mut state = self.state.lock(); + let slot = cell.slot.load(Ordering::Acquire); + if slot == NO_RENDER_ID { return; } + let index = usize::from(slot - render_ids::FIRST); + if state.cells[index].as_ref().is_some_and(|owner| Arc::ptr_eq(owner, cell)) + && state.leases.remove(cell.owner, |_| true).is_ok() { + let old = state.cells[index].take(); + cell.slot.store(NO_RENDER_ID, Ordering::Release); + drop(state); + drop(old); + self.available.notify(); } } + + /// Caller holds the device mutex and supplies the exact final release proof. + pub(crate) fn reclaim_one(&self, released: impl FnMut(u16) -> bool) -> Option { + self.reclaim(false, released) + } + /// Reuse backing for a pool that already has an ID, without evicting an + /// empty metadata-only owner. Caller supplies the same release proof. + pub(crate) fn reclaim_backing(&self, released: impl FnMut(u16) -> bool) -> Option { + self.reclaim(true, released) + } + fn reclaim(&self, only_storage: bool, mut released: impl FnMut(u16) -> bool) -> Option { + let mut state = self.state.lock(); + let RenderIdState { leases, cells } = &mut *state; + let mut select = |require_storage: bool| leases.reclaim_one(|lease| { + cells[usize::from(lease.slot - render_ids::FIRST)].as_ref().is_some_and(|cell| + cell.owner == lease.owner + && cell.slot.load(Ordering::Acquire) == lease.slot + && cell.generation.load(Ordering::Acquire) == lease.generation + && (!require_storage || cell.has_storage())) + && released(lease.slot) + }); + let lease = select(true).or_else(|| if only_storage { None } else { select(false) })?; + let old = state.cells[usize::from(lease.slot - render_ids::FIRST)].take(); + let (backing, spare) = if let Some(cell) = old.as_ref() { + let backing = cell.storage.backing.lock().take(); + let spare = cell.storage.spare.lock().take(); + cell.storage.page_list.store(0, Ordering::Release); + cell.slot.store(NO_RENDER_ID, Ordering::Release); + (backing, spare) + } else { (None, None) }; + drop(state); + drop(old); + self.available.notify(); + Some(ReclaimedRenderId { _backing: backing, spare }) + } +} + +pub(super) struct RenderIdPin { + allocator: Arc, + cell: Arc, + lease: render_ids::Lease, +} +impl Drop for RenderIdPin { + fn drop(&mut self) { + let available = self.allocator.state.lock().leases.unpin(self.lease).unwrap_or(false); + if self.cell.closed.load(Ordering::Acquire) { self.allocator.close(&self.cell); } + if available { self.allocator.available.notify(); } + } +} + +enum RenderIdKind { + Fixed(u16), + Rebindable(Arc), } -struct RenderId { - slot: u16, +pub(super) struct RenderId { + kind: RenderIdKind, allocator: Arc, } impl RenderId { - fn new(allocator: &Arc) -> Result { + pub(super) fn slot(&self) -> u16 { + match &self.kind { + RenderIdKind::Fixed(slot) => *slot, + RenderIdKind::Rebindable(cell) => cell.slot.load(Ordering::Acquire), + } + } + pub(super) fn logical(allocator: &Arc) -> Result { + let owner = NEXT_RENDER_OWNER.fetch_update(Ordering::Relaxed, Ordering::Relaxed, + |next| next.checked_add(1)).map_err(|_| EOVERFLOW)?; + let storage = RenderStorage::new(None)?; + Ok(Self { kind: RenderIdKind::Rebindable(Arc::new(RenderIdCell { + owner, slot: AtomicU16::new(NO_RENDER_ID), generation: AtomicU64::new(0), + closed: AtomicBool::new(false), + storage, + }, GFP_KERNEL)?), allocator: allocator.clone() }) + } + fn storage(&self) -> Option> { + match &self.kind { + RenderIdKind::Rebindable(cell) => Some(cell.storage.clone()), + RenderIdKind::Fixed(_) => None, + } + } + fn pin(&self, assigned: impl FnOnce()) -> Result> { + match &self.kind { + RenderIdKind::Fixed(_) => Ok(None), + RenderIdKind::Rebindable(cell) => self.allocator.pin(cell, assigned).map(Some), + } + } + pub(super) fn new(allocator: &Arc) -> Result { for (bank, used) in allocator.used.iter().enumerate() { let mut value = used.load(Ordering::Acquire); loop { @@ -51,7 +226,7 @@ impl RenderId { ) { Ok(_) => { return Ok(Self { - slot: 129 + bank as u16 * 64 + bit as u16, + kind: RenderIdKind::Fixed(129 + bank as u16 * 64 + bit as u16), allocator: allocator.clone(), }) } @@ -64,11 +239,39 @@ impl RenderId { } impl Drop for RenderId { fn drop(&mut self) { - let bit = self.slot as usize - 129; - self.allocator.used[bit / 64].fetch_and(!(1u64 << (bit % 64)), Ordering::Release); + match &self.kind { + RenderIdKind::Fixed(slot) => { + let bit = *slot as usize - 129; + self.allocator.used[bit / 64].fetch_and(!(1u64 << (bit % 64)), Ordering::Release); + } + RenderIdKind::Rebindable(cell) => self.allocator.close(cell), + } } } +/// What one command needs of its compute queue's USC pool, in the family's +/// terms: G17P pools count blocks, M5 pools have the command plan's geometry +/// (`None`: the default M5 pool). Derived once per command +/// (`Backend::compute_capacity`); every admission check and allocation takes it. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum ComputeCapacity { + Blocks(usize), + M5(Option), +} + +/// Profile-specific descriptor data for a fresh physical pool. The runtime +/// installs this only before publication, or after an acknowledged release. +pub(crate) enum PoolDescriptor { + Neo { slot: u16, page_list: u64, blocks: usize }, + M5 { slot: u16, words: [u64; 8] }, +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum PageLayout { + Neo { blocks: usize, runs: RunLayout }, + M5(super::fw::m5::pool::PoolGeometry), +} + /// Free-list backing is by far the largest user of the shared lower GPU VA /// window, and placing it populates every page before the window is searched. /// After a refused placement, builders waiting for capacity would allocate and @@ -125,8 +328,10 @@ fn place_lower(size: usize, place: impl FnOnce() -> Result) -> Result { pub(crate) struct Pages { backing: KernelObject, - blocks: usize, - layout: RunLayout, + layout: PageLayout, + /// The charge of an M5 backing sized by a client's request, released with it. It is the + /// backing's only count: the backing itself is not counted as kernel memory. + charge: Option, } impl Drop for Pages { fn drop(&mut self) { @@ -134,7 +339,7 @@ impl Drop for Pages { } } impl Pages { - fn new(alloc: &Allocator<'_>, blocks: usize, layout: RunLayout) -> Result { + fn new(alloc: &Allocator<'_>, blocks: usize, runs: RunLayout) -> Result { let size = TABLE_SIZE + blocks * FREE_LIST_BLOCK_SIZE; let backing = place_lower(size, || { alloc.lower( @@ -144,39 +349,148 @@ impl Pages { CpuMap::WriteCombined, ) })?; - Self::from_backing(backing, blocks, layout) + Self::from_backing(backing, blocks, runs) } - fn from_backing(mut backing: KernelObject, blocks: usize, layout: RunLayout) -> Result { + fn from_backing(mut backing: KernelObject, blocks: usize, runs: RunLayout) -> Result { let base = backing.gpu_va(); - backing.initialize::(0, |tables| tables.init(base, blocks, layout))?; + backing.initialize::(0, |tables| tables.init(base, blocks, runs))?; Ok(Self { backing, - blocks, - layout, + layout: PageLayout::Neo { blocks, runs }, + charge: None, }) } - /// Fresh compute backing at the admitted capacity, built off-lock. - pub(crate) fn compute_sized(alloc: &Allocator<'_>, blocks: usize) -> Result { + /// Fresh compute backing at the admitted capacity, built off-lock, in the + /// capacity's family format. + /// An M5 backing is charged to `client` when one is given (see [`Self::m5`]). + pub(crate) fn for_compute(alloc: &Allocator<'_>, capacity: ComputeCapacity, + client: Option<&Arc>) -> Result { + match capacity { + ComputeCapacity::Blocks(blocks) => Self::compute_sized(alloc, blocks), + ComputeCapacity::M5(geometry) => Self::compute_m5(alloc, geometry, client), + } + } + fn compute_sized(alloc: &Allocator<'_>, blocks: usize) -> Result { if blocks < COMPUTE_BLOCKS || blocks > PAGE_LIST_SIZE / (FREE_LIST_BLOCK_PAGES as usize * 8) { return Err(EINVAL); } Self::new(alloc, blocks, RunLayout::Compute) } - pub(crate) fn map_client(&self, vm: &mmu::Vm) -> Result { + /// Fresh zeroed M5 compute backing, built without the device mutex. M5 keeps + /// its directory, block list and initial eight render blocks in one retained + /// allocation. + fn compute_m5(alloc: &Allocator<'_>, required: Option, + client: Option<&Arc>) -> Result { + use super::fw::m5::pool; + let geometry = match required { Some(geometry) => geometry, + None => pool::PoolGeometry::g17s(8).map_err(|_| EINVAL)?, }; + Self::m5(alloc, geometry, client) + } + /// M5 pool sizes follow the client's program records, up to the board's whole inventory, + /// so the backing is admitted against `client`'s and the device's limits before it is + /// allocated, as a user object of that size would be, and counted as that client's held + /// memory instead of kernel memory. A refusal fails with ENOMEM. + fn m5(alloc: &Allocator<'_>, geometry: super::fw::m5::pool::PoolGeometry, + client: Option<&Arc>) -> Result { + let tables = geometry.directory_bytes.checked_add(geometry.freelist_bytes).ok_or(EOVERFLOW)?; + let size = geometry.blocks.checked_mul(geometry.block_bytes()) + .and_then(|n| n.checked_add(tables)).ok_or(EOVERFLOW)?; + let (backing, charge) = match client { + Some(client) => { + let commit = crate::memacct::LEDGER.admit(client, size as u64)?; + let backing = alloc.lower_user_charged(size, mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_SHARED_RW)?; + let charge = crate::memacct::Charge::user(commit, None); + charge.set_held(true); + (backing, Some(charge)) + } + None => (alloc.lower(size, mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_SHARED_RW, CpuMap::WriteCombined)?, None), + }; + let mut pages = Self { backing, layout: PageLayout::M5(geometry), charge }; + pages.populate_m5()?; + Ok(pages) + } + fn populate_m5(&mut self) -> Result { + use super::fw::m5::pool; + let PageLayout::M5(geometry) = self.layout else { return Err(EINVAL); }; + let tables = geometry.directory_bytes.checked_add(geometry.freelist_bytes).ok_or(EOVERFLOW)?; + let base = self.backing.gpu_va(); + let mut blocks = KVec::with_capacity(geometry.blocks, GFP_KERNEL)?; + for index in 0..geometry.blocks { + blocks.push(base + (tables + index * geometry.block_bytes()) as u64, GFP_KERNEL)?; + } + pool::encode_directory(geometry, &blocks, + &mut self.backing.bytes_mut()[..geometry.directory_bytes]).map_err(|_| EINVAL)?; + pool::encode_freelist(geometry, &blocks, + &mut self.backing.bytes_mut()[geometry.directory_bytes..tables]).map_err(|_| EINVAL)?; + Ok(()) + } + fn reset_for_client(&mut self) -> Result { + // Released storage may contain the previous client's private data. + // Clear it before exposing an alias in a different client VM. + self.backing.bytes_mut().fill(0); + self.populate_m5()?; + fence(Ordering::SeqCst); + Ok(()) + } + /// Aliases the backing into `vm`, whose client becomes its user: a charged backing's + /// charge moves to that client (ENOMEM over its limit), so storage that another client + /// released and this one reuses is no longer counted against the first. + pub(crate) fn map_client(&mut self, vm: &mmu::Vm) -> Result { + if let (Some(charge), Some(client)) = (self.charge.as_mut(), vm.memory_client()) { + charge.transfer(client)?; + } self.backing.map_alias(vm, self.page_list_va(), mmu::PROT_GPU_SHARED_RW) } - pub(crate) fn page_list_va(&self) -> u64 { - self.backing.gpu_va() + fn fits(&self, required: Option) -> bool { + match (self.layout, required) { + (PageLayout::M5(current), Some(required)) => + current.block_bytes() == required.block_bytes() && current.blocks >= required.blocks, + (_, None) => true, + _ => false, + } + } + /// Whether this backing holds `required`; backing of the other family never does. + pub(crate) fn fits_capacity(&self, required: ComputeCapacity) -> bool { + match (self.layout, required) { + (PageLayout::Neo { blocks, .. }, ComputeCapacity::Blocks(required)) => blocks >= required, + (PageLayout::M5(_), ComputeCapacity::M5(geometry)) => self.fits(geometry), + _ => false, + } } + fn is_m5(&self) -> bool { matches!(self.layout, PageLayout::M5(_)) } + pub(crate) fn page_list_va(&self) -> u64 { self.backing.gpu_va() } fn run_list_va(&self) -> u64 { - self.page_list_va() + PAGE_LIST_SIZE as u64 + self.page_list_va() + match self.layout { + PageLayout::Neo { .. } => PAGE_LIST_SIZE, + PageLayout::M5(g) => g.directory_bytes, + } as u64 + } + fn m5_layout(&self, slot: u16) -> Result { + let PageLayout::M5(geometry) = self.layout else { return Err(EINVAL); }; + Ok(super::fw::m5::pool::PoolLayout { geometry, directory: self.page_list_va(), + freelist: self.run_list_va(), hardware_id: u32::from(slot) }) + } + /// The descriptor row of this backing installed at `slot`. + pub(crate) fn descriptor(&self, slot: u16) -> Result { + match self.layout { + PageLayout::M5(_) => { + let words = super::fw::m5::pool::initial_uma_descriptor(self.m5_layout(slot)?) + .map_err(|_| EINVAL)?; + Ok(PoolDescriptor::M5 { slot, words }) + } + PageLayout::Neo { blocks, .. } => + Ok(PoolDescriptor::Neo { slot, page_list: self.page_list_va(), blocks }), + } } fn repopulate(&self) -> Result { + let PageLayout::Neo { blocks, runs } = self.layout else { return Err(EINVAL); }; let table = self.backing.pointer(0, TABLE_SIZE)?.cast::(); PoolTables::for_each_word( self.page_list_va(), - self.blocks, - self.layout, + blocks, + runs, |index, value| { // SAFETY: The page-aligned backing was validated for the complete // tables. The enumerator bounds every index inside those tables; @@ -194,11 +508,17 @@ static RENDER_POOL_SERIAL: AtomicU64 = AtomicU64::new(0); /// both aliases before the backing. pub(crate) struct RenderBacking { _client: mmu::KernelMapping, - _global: mmu::KernelMapping, + _global: Option, pages: Pages, } impl RenderBacking { + fn into_pages(self) -> Pages { + let Self { _client, _global, pages } = self; + drop(_client); + drop(_global); + pages + } /// Fresh populated backing, built without the device mutex. pub(crate) fn new( alloc: &Allocator<'_>, @@ -222,10 +542,37 @@ impl RenderBacking { let pages = Pages::from_backing(backing, blocks, RunLayout::Render)?; Ok(Self { _client: client, - _global: global, + _global: Some(global), pages, }) } + /// Fresh M5 backing, aliased into the client VM only: M5 firmware uses + /// the canonical and client mappings of these pages. + fn m5(alloc: &Allocator<'_>, client: &mmu::Vm, + geometry: super::fw::m5::pool::PoolGeometry) -> Result { + let mut pages = Pages::m5(alloc, geometry, client.memory_client())?; + let client = pages.map_client(client)?; + Ok(Self { _client: client, _global: None, pages }) + } +} + +#[pin_data] +struct RenderStorage { + #[pin] + backing: Mutex>, + #[pin] + spare: Mutex>, + page_list: AtomicU64, +} +impl RenderStorage { + fn new(backing: Option) -> Result> { + let address = backing.as_ref().map_or(0, |backing| backing.pages.page_list_va()); + Arc::pin_init(try_pin_init!(Self { + backing <- new_mutex!(backing, "G17 render pool backing"), + spare <- new_mutex!(None, "G17 render pool spare"), + page_list: AtomicU64::new(address), + }), GFP_KERNEL) + } } /// A render pool retained by a logical context and every graph referencing it. @@ -233,23 +580,39 @@ impl RenderBacking { /// consumed and the client VM is evicted; a fresh backing is installed before /// the pool is staged again. pub(crate) struct RenderPool { - blocks: usize, - backing: Arc>>, - /// Page-list VA of the installed backing; zero while vacant. - page_list: AtomicU64, + storage: Arc, /// Counts installed backings, so a re-armed backing is a new free-list /// owner even if its addresses repeat. epoch: AtomicU64, /// Residency of the client VM the backing is aliased into. client_residency: Option>, + /// The one VM (`mmu::Vm::id`) this pool's backing is aliased into. + client_vm: u64, control: KernelObject, state: KernelObject, served: AtomicBool, serial: u64, id: RenderId, + layout: PageLayout, +} + +/// Keeps both a physical identity and its logical pool alive during use. +/// Drop the identity pin before releasing the retained pool reference. +pub(crate) struct PoolPin { + _identity: RenderIdPin, + _pool: Arc, } impl RenderPool { + /// The caller retains this guard across construction and all firmware use. + pub(crate) fn pin_id(self: &Arc) -> Result> { + let pin = self.id.pin(|| { + self.epoch.fetch_add(1, Ordering::AcqRel); + self.served.store(true, Ordering::Release); + })?; + Ok(pin.map(|pin| PoolPin { _identity: pin, _pool: self.clone() })) + } + pub(crate) fn new( alloc: &Allocator<'_>, ids: &Arc, @@ -259,6 +622,32 @@ impl RenderPool { ) -> Result { let id = RenderId::new(ids)?; let backing = RenderBacking::new(alloc, render_global, client, blocks)?; + Self::with_backing(alloc, id, client, backing) + } + + /// Explicit initial geometry; allocation policy remains with the caller. + /// M5 firmware uses the canonical and client mappings of these pages. + pub(super) fn m5(alloc: &Allocator<'_>, id: RenderId, client: &mmu::Vm, + geometry: super::fw::m5::pool::PoolGeometry) -> Result { + if !alloc.uat.is_m5() { return Err(EINVAL); } + // Logical metadata is cheap; physical backing is installed only once + // preparation owns both an identity pin and admission capacity. + let storage = id.storage().ok_or(EINVAL)?; + let state = alloc.kernel(CONTROL_SIZE, mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_FW_SHARED_RW, CpuMap::WriteCombined)?; + let control = alloc.kernel(CONTROL_SIZE, mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_FW_SHARED_RW, CpuMap::WriteCombined)?; + let serial = RENDER_POOL_SERIAL.fetch_update(Ordering::Relaxed, Ordering::Relaxed, + |value| value.checked_add(1)).map_err(|_| EOVERFLOW)? + 1; + Ok(Self { storage, epoch: AtomicU64::new(0), client_residency: client.residency_gate(), + client_vm: client.id(), + control, state, served: AtomicBool::new(true), serial, id, + layout: PageLayout::M5(geometry) }) + } + + fn with_backing(alloc: &Allocator<'_>, id: RenderId, client: &mmu::Vm, + backing: RenderBacking) -> Result { + let layout = backing.pages.layout; let address = backing.pages.page_list_va(); let run_list_va = backing.pages.run_list_va(); let state = alloc.kernel( @@ -273,45 +662,51 @@ impl RenderPool { mmu::PROT_GPU_FW_SHARED_RW, CpuMap::WriteCombined, )?; - control.write( - 0, - FreeListControl::render( - &FreeListArgs { - buffer_slot: u32::from(id.slot), - class: FreeListClass::Two, - page_list_va: address, - run_list_va, - blocks: blocks as u32, - state_va: state.gpu_va(), - }, + match layout { + PageLayout::M5(_) => { + super::fw::m5::pool::encode_g17s_render_state(backing.pages.m5_layout(id.slot())?, + state.gpu_va(), control.bytes_mut()).map_err(|_| EINVAL)?; + } + PageLayout::Neo { blocks, .. } => control.write( 0, - ), - )?; + FreeListControl::render( + &FreeListArgs { + buffer_slot: u32::from(id.slot()), + class: FreeListClass::Two, + page_list_va: address, + run_list_va, + blocks: blocks as u32, + state_va: state.gpu_va(), + }, + 0, + ), + )?, + } fence(Ordering::SeqCst); let serial = RENDER_POOL_SERIAL .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |value| value.checked_add(1)) .map_err(|_| EOVERFLOW)? + 1; - let backing = Arc::pin_init( - new_mutex!(Some(backing), "G17 render pool backing"), - GFP_KERNEL, - )?; + let storage = RenderStorage::new(Some(backing))?; Ok(Self { - blocks, - backing, - page_list: AtomicU64::new(address), + storage, epoch: AtomicU64::new(0), client_residency: client.residency_gate(), + client_vm: client.id(), control, state, served: AtomicBool::new(false), serial, id, + layout, }) } /// No backing is installed: the pool must be re-armed before staging. pub(crate) fn vacant(&self) -> bool { - self.page_list.load(Ordering::Acquire) == 0 + self.storage.page_list.load(Ordering::Acquire) == 0 + } + pub(crate) fn needs_storage(&self) -> bool { + self.vacant() && self.storage.spare.lock().is_none() } /// Pressure reclaim evicted the client VM, which is idle. pub(crate) fn client_evicted(&self) -> bool { @@ -322,9 +717,9 @@ impl RenderPool { /// next staging requires a re-armed backing. The caller drops the result /// after releasing the device mutex. pub(crate) fn vacate(&self) -> Option { - let backing = self.backing.lock().take(); + let backing = self.storage.backing.lock().take(); if backing.is_some() { - self.page_list.store(0, Ordering::Release); + self.storage.page_list.store(0, Ordering::Release); } backing } @@ -332,7 +727,7 @@ impl RenderPool { /// The next staging rewrites the control header and descriptor row for it /// (its run-list address changes the free-list owner identity). pub(crate) fn rearm(&self, fresh: RenderBacking) -> Result { - let mut backing = self.backing.lock(); + let mut backing = self.storage.backing.lock(); if backing.is_some() { return Ok(()); } @@ -340,21 +735,63 @@ impl RenderPool { *backing = Some(fresh); self.epoch.fetch_add(1, Ordering::AcqRel); self.served.store(true, Ordering::Release); - self.page_list.store(address, Ordering::Release); + self.storage.page_list.store(address, Ordering::Release); Ok(()) } + /// Keep released physical storage for the incoming owner, without keeping + /// the old VM alias or assigning backing to every logical context. Its + /// charge moves to the owner's client when the owner maps it + /// ([`Pages::map_client`]). + pub(crate) fn cache_reclaimed(&self, reclaimed: ReclaimedRenderId) { + let pages = reclaimed.into_pages(); + let backing = self.storage.backing.lock(); + let mut spare = self.storage.spare.lock(); + let mut pages = pages; + if backing.is_none() && spare.is_none() + && pages.as_ref().is_some_and(|pages| pages.layout == self.layout) { + *spare = pages.take(); + } + drop(spare); + drop(backing); + drop(pages); + } /// Re-arms a vacant pool with fresh backing built here, without the device - /// mutex. `client` is the VM of the context that owns this pool. + /// mutex. `client` is the VM of the context that owns this pool; EPERM + /// for any other VM. pub(crate) fn ensure_backing( &self, alloc: &Allocator<'_>, render_global: &mmu::Vm, client: &mmu::Vm, ) -> Result { - if !self.vacant() { - return Ok(()); + if client.id() != self.client_vm { + return Err(EPERM); } - self.rearm(RenderBacking::new(alloc, render_global, client, self.blocks)?) + let geometry = match self.layout { + PageLayout::M5(geometry) => geometry, + PageLayout::Neo { blocks, .. } => { + if !self.vacant() { + return Ok(()); + } + return self.rearm(RenderBacking::new(alloc, render_global, client, blocks)?); + } + }; + let mut backing = self.storage.backing.lock(); + if backing.is_some() { return Ok(()); } + let spare = self.storage.spare.lock().take(); + let fresh = if let Some(mut pages) = spare { + pages.reset_for_client()?; + let alias = pages.map_client(client)?; + RenderBacking { _client: alias, _global: None, pages } + } else { + RenderBacking::m5(alloc, client, geometry)? + }; + let address = fresh.pages.page_list_va(); + *backing = Some(fresh); + self.epoch.fetch_add(1, Ordering::AcqRel); + self.served.store(true, Ordering::Release); + self.storage.page_list.store(address, Ordering::Release); + Ok(()) } pub(crate) fn serial(&self) -> u64 { @@ -364,21 +801,26 @@ impl RenderPool { self.epoch.load(Ordering::Acquire) } pub(crate) fn id(&self) -> u16 { - self.id.slot + self.id.slot() } pub(crate) fn control_va(&self) -> u64 { self.control.gpu_va() } /// Zero while vacant. pub(crate) fn page_list_va(&self) -> u64 { - self.page_list.load(Ordering::Acquire) + self.storage.page_list.load(Ordering::Acquire) } pub(crate) fn run_list_va(&self) -> u64 { - self.page_list_va() + PAGE_LIST_SIZE as u64 + self.page_list_va() + match self.layout { + PageLayout::Neo { .. } => PAGE_LIST_SIZE, + PageLayout::M5(g) => g.directory_bytes, + } as u64 } pub(crate) fn end_va(&self) -> u64 { - self.run_list_va() - + (self.blocks * size_of::()) as u64 + self.run_list_va() + match self.layout { + PageLayout::Neo { blocks, .. } => blocks * size_of::(), + PageLayout::M5(g) => g.blocks * 8, + } as u64 } /// Fresh pools are already populated. Reuse requires the previous release @@ -388,13 +830,34 @@ impl RenderPool { return Ok(()); } let submitted = self.state.dword(0)?.load(Ordering::Relaxed); + let blocks = match self.layout { + PageLayout::Neo { blocks, .. } => blocks, + PageLayout::M5(_) => { + let backing = self.storage.backing.lock(); + let pages = &backing.as_ref().ok_or(EAGAIN)?.pages; + if pages.page_list_va() != self.page_list_va() { return Err(EIO); } + let mut bytes = [0; super::fw::m5::pool::STATE_BYTES]; + super::fw::m5::pool::encode_g17s_render_state(pages.m5_layout(self.id())?, + self.state.gpu_va(), &mut bytes).map_err(|_| EINVAL)?; + bytes[super::fw::m5::pool::COMPLETED..super::fw::m5::pool::COMPLETED + 8] + .copy_from_slice(&submitted.to_le_bytes()); + let raw = self.control.pointer(0, bytes.len())?; + for (index, byte) in bytes.into_iter().enumerate() { + // SAFETY: The whole retained range was checked and the caller + // observed final release before installing fresh pool backing. + unsafe { raw.add(index).write_volatile(byte) }; + } + fence(Ordering::SeqCst); + return Ok(()); + } + }; let header = FreeListControl::render( &FreeListArgs { buffer_slot: u32::from(self.id()), class: FreeListClass::Two, page_list_va: self.page_list_va(), run_list_va: self.run_list_va(), - blocks: self.blocks as u32, + blocks: blocks as u32, state_va: self.state.gpu_va(), }, submitted, @@ -409,7 +872,7 @@ impl RenderPool { .write_unaligned(header) }; let control = self.control.pointer(0, CONTROL_SIZE)?.cast::(); - let backing = self.backing.lock(); + let backing = self.storage.backing.lock(); let pages = &backing.as_ref().ok_or(EAGAIN)?.pages; if pages.page_list_va() != self.page_list_va() { return Err(EIO); @@ -428,7 +891,9 @@ impl RenderPool { /// counter, so retain the byte accesses required by that interface. pub(crate) fn counters(&self) -> Result<[u64; 2]> { let submitted = self.state.dword(0)?.load(Ordering::Relaxed); - let completed = self.control.pointer(FreeListControl::COMPLETED, 8)?; + let offset = if self.is_m5() { super::fw::m5::pool::COMPLETED } + else { FreeListControl::COMPLETED }; + let completed = self.control.pointer(offset, 8)?; let mut bytes = [0; 8]; for (index, byte) in bytes.iter_mut().enumerate() { // SAFETY: The checked eight-byte range stays mapped; byte accesses @@ -438,6 +903,47 @@ impl RenderPool { Ok([submitted, u64::from_le_bytes(bytes)]) } + pub(crate) fn fits(&self, geometry: Option) -> bool { + match (self.layout, geometry) { + (PageLayout::M5(current), Some(required)) => + current.block_bytes() == required.block_bytes() && current.blocks >= required.blocks, + (PageLayout::Neo { .. }, None) => true, + _ => false, + } + } + + pub(crate) fn is_m5(&self) -> bool { matches!(self.layout, PageLayout::M5(_)) } + + fn m5_state(&self) -> Result { + if !self.is_m5() { return Err(EINVAL); } + let mut state = super::fw::m5::pool::PoolState([0; super::fw::m5::pool::STATE_BYTES]); + let raw = self.control.pointer(0, state.0.len())?; + fence(Ordering::Acquire); + for (index, byte) in state.0.iter_mut().enumerate() { + // SAFETY: The entire retained header range was checked above. + *byte = unsafe { raw.add(index).read_volatile() }; + } + Ok(state) + } + + pub(crate) fn m5_descriptor(&self) -> Result<[u64; 8]> { Ok(self.m5_state()?.descriptor()) } + + pub(crate) fn store_m5_descriptor(&self, words: [u64; 8]) -> Result { + let mut state = self.m5_state()?; + if !state.matches(&words) { return Err(EIO); } + state.restore(words); + for offset in super::fw::m5::pool::SAVED_FIELDS { + let raw = self.control.pointer(offset, 4)?; + for index in 0..4 { + // SAFETY: Each mutable field was checked; the caller owns the + // retired descriptor, and immutable header bytes stay intact. + unsafe { raw.add(index).write_volatile(state.0[offset + index]) }; + } + } + fence(Ordering::SeqCst); + Ok(()) + } + /// Snapshot the unaligned shared fields before loading their descriptor. pub(crate) fn page_pool_state(&self) -> Result { let raw = self.control.pointer(0x14, 0x1c)?; @@ -511,23 +1017,39 @@ impl Submitted<'_> { /// Idle retained queues may give their USC backing back (`pages` is `None`) /// once the firmware consumed the pool's release; the control and state /// objects the queue record names stay until processor stop. +/// +/// A backing serves one VM: its blocks hold the stack and scratch memory of +/// that VM's work, so a view of it in another VM is refused +/// ([`Self::map_client`]). Another VM gets the queue only after the backing +/// was given back and fresh, zeroed backing installed ([`Self::rearm`]). pub(crate) struct ComputePool { pages: Option, + /// The VM (`mmu::Vm::id`) the backing serves, from its first client view + /// until it is given back; zero while no client has had a view of it. + client_vm: u64, control: KernelObject, state: KernelObject, id: u16, + m5: bool, } impl ComputePool { - pub(crate) fn fits(&self, blocks: usize) -> bool { - self.pages.as_ref().is_some_and(|pages| pages.blocks >= blocks) + /// Capacity check for admission and growth. + pub(crate) fn fits_capacity(&self, required: ComputeCapacity) -> bool { + self.pages.as_ref().is_some_and(|pages| pages.fits_capacity(required)) } - pub(crate) fn new(alloc: &Allocator<'_>, qid: u8, blocks: usize) -> Result { + /// A pool at the admitted capacity, in the capacity's family format. + pub(crate) fn new(alloc: &Allocator<'_>, qid: u8, required: ComputeCapacity, + client: Option<&Arc>) -> Result { if qid > super::fw::kick::QID_MAX { return Err(EINVAL); } + let pages = Pages::for_compute(alloc, required, client)?; + let m5 = pages.is_m5(); + Self::with_pages(alloc, qid, m5, pages) + } + fn with_pages(alloc: &Allocator<'_>, qid: u8, m5: bool, pages: Pages) -> Result { let id = u16::from(qid) + 1; - let pages = Pages::compute_sized(alloc, blocks)?; let mut control = alloc.kernel( CONTROL_SIZE, mmu::UAT_PGSZ as u64, @@ -540,38 +1062,64 @@ impl ComputePool { mmu::PROT_GPU_FW_SHARED_RW, CpuMap::WriteCombined, )?; - control.write( - 0, - FreeListControl::compute(&FreeListArgs { - buffer_slot: u32::from(id), - class: FreeListClass::One, - page_list_va: pages.page_list_va(), - run_list_va: pages.run_list_va(), - blocks: blocks as u32, - state_va: state.gpu_va(), - }), - )?; + Self::initialize_control(&mut control, &pages, id, state.gpu_va(), [0; 8])?; fence(Ordering::SeqCst); - Ok(Self { - pages: Some(pages), - control, - state, - id, - }) + Ok(Self { pages: Some(pages), client_vm: 0, control, state, id, m5 }) + } + fn initialize_control(control: &mut KernelObject, pages: &Pages, id: u16, + submitted: u64, completed: [u8; 8]) -> Result { + match pages.layout { + PageLayout::M5(_) => { + use super::fw::m5::pool; + pool::encode_g17s_compute_state(pages.m5_layout(id)?, submitted, + &mut control.bytes_mut()[..pool::STATE_BYTES]).map_err(|_| EINVAL)?; + control.bytes_mut()[pool::COMPLETED..pool::COMPLETED + 8].copy_from_slice(&completed); + Ok(()) + } + PageLayout::Neo { blocks, .. } => { + let mut header = FreeListControl::compute(&FreeListArgs { + buffer_slot: u32::from(id), class: FreeListClass::One, + page_list_va: pages.page_list_va(), run_list_va: pages.run_list_va(), + blocks: blocks as u32, state_va: submitted, + }); + header.unk_54[..8].copy_from_slice(&completed); + control.write(0, header) + } + } + } + pub(crate) fn descriptor(&self) -> Result { + self.pages.as_ref().ok_or(EAGAIN)?.descriptor(self.id) + } + pub(crate) fn completed_submissions(&self) -> Result { + Ok(u64::from_le_bytes(self.completed_bytes()?)) + } + fn completed_bytes(&self) -> Result<[u8; 8]> { + let offset = if self.m5 { super::fw::m5::pool::COMPLETED } else { FreeListControl::COMPLETED }; + let pointer = self.control.pointer(offset, 8)?; + let mut bytes = [0; 8]; + for (index, byte) in bytes.iter_mut().enumerate() { + // SAFETY: The checked packed counter remains mapped. Firmware may + // update it; callers use it only with the queue's completion proof. + *byte = unsafe { pointer.add(index).read_volatile() }; + } + Ok(bytes) } pub(crate) fn id(&self) -> u16 { self.id } - /// EAGAIN while the pool is vacant: the owner must re-arm it first. - pub(crate) fn page_list_va(&self) -> Result { - self.pages.as_ref().map(Pages::page_list_va).ok_or(EAGAIN) - } pub(crate) fn control_va(&self) -> u64 { self.control.gpu_va() } + pub(crate) fn fits(&self, required: Option) -> bool { + self.pages.as_ref().is_some_and(|pages| pages.fits(required)) + } pub(crate) fn populated(&self) -> bool { self.pages.is_some() } + /// Whether a view of the backing may be given to `vm`. + pub(crate) fn serves(&self, vm: &mmu::Vm) -> bool { + self.client_vm == 0 || self.client_vm == vm.id() + } /// The release record for this pool's buffer slot under the free-list /// generation the queue last published. pub(crate) fn release_record(&self, generation: u64) -> super::fw::channels::FreeListRelease { @@ -580,6 +1128,7 @@ impl ComputePool { /// The caller has observed the firmware consume this pool's release and /// frees the returned backing after releasing the device mutex. pub(crate) fn take_pages(&mut self) -> Option { + self.client_vm = 0; self.pages.take() } /// Rewrites the shared control header for fresh backing. The caller then @@ -596,39 +1145,29 @@ impl ComputePool { } /// Preserve both sets of pages on failure. The caller owns the consumed /// release and returns old pages for destruction outside the device mutex. + /// The grown backing's alias is in the VM the old backing served, which it + /// keeps serving. pub(crate) fn replace_pages(&mut self, pages: &mut Option) -> Result { self.write_pages(pages.as_ref().ok_or(EINVAL)?)?; core::mem::swap(&mut self.pages, pages); Ok(()) } fn write_pages(&mut self, pages: &Pages) -> Result { - let completed = self.control.pointer(FreeListControl::COMPLETED, 8)?; - let mut bytes = [0; 8]; - for (index, byte) in bytes.iter_mut().enumerate() { - // SAFETY: The checked eight-byte range stays mapped; byte accesses - // match the unaligned packed counter the firmware writes. - *byte = unsafe { completed.add(index).read_volatile() }; - } - let mut header = FreeListControl::compute(&FreeListArgs { - buffer_slot: u32::from(self.id), - class: FreeListClass::One, - page_list_va: pages.page_list_va(), - run_list_va: pages.run_list_va(), - blocks: pages.blocks as u32, - state_va: self.state.gpu_va(), - }); - header.unk_54[..8].copy_from_slice(&bytes); - self.control.write(0, header)?; + if pages.is_m5() != self.m5 { return Err(EINVAL); } + let completed = self.completed_bytes()?; + Self::initialize_control(&mut self.control, pages, self.id, self.state.gpu_va(), completed)?; fence(Ordering::SeqCst); Ok(()) } + /// EPERM for a VM other than the one the backing serves. pub(crate) fn map_client(&mut self, vm: &mmu::Vm) -> Result { - let pages = self.pages.as_mut().ok_or(EAGAIN)?; - let address = pages.page_list_va(); - pages - .backing - .map_alias(vm, address, mmu::PROT_GPU_SHARED_RW) + if !self.serves(vm) { + return Err(EPERM); + } + let alias = self.pages.as_mut().ok_or(EAGAIN)?.map_client(vm)?; + self.client_vm = vm.id(); + Ok(alias) } pub(crate) fn reserve_submission(&self) -> Result { let counter = self.state.dword(0)?; @@ -826,6 +1365,11 @@ impl Lease { self.phase = LeasePhase::ReleaseSent(cursor); Ok(()) } + /// Only sent releases and inherited release witnesses consume cursors. + pub(crate) fn needs_release_observation(&self) -> bool { + self.previous_release.is_some() || matches!(self.phase, LeasePhase::ReleaseSent(_)) + } + /// Observes all older release records passed by one consumer snapshot. pub(crate) fn observe_releases(&mut self, consumer: u32, producer: u32) -> Result { if let Some(cursor) = self.previous_release { @@ -867,7 +1411,7 @@ mod capacity_tests { #[test] fn rejected_pool_construction_returns_all_render_ids() { - let ids = Arc::new(RenderIds::new(), GFP_KERNEL).unwrap(); + let ids = RenderIds::new().unwrap(); for _ in 0..8 { let mut pools = KVec::with_capacity(127, GFP_KERNEL).unwrap(); for _ in 0..127 { diff --git a/drivers/gpu/drm/asahi/g17/freelist/render_ids.rs b/drivers/gpu/drm/asahi/g17/freelist/render_ids.rs new file mode 100644 index 0000000000000..e750b01bd91c7 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/freelist/render_ids.rs @@ -0,0 +1,246 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Pure physical-ID lease rules. The caller serializes this state and supplies +//! a firmware release proof before a slot is reassigned to another pool. + +pub(super) const FIRST: u16 = 129; +pub(super) const COUNT: usize = 127; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) struct Lease { + pub(super) slot: u16, + pub(super) owner: u64, + pub(super) generation: u64, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) struct Claim { + pub(super) lease: Lease, + pub(super) displaced: Option, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum Error { + BadOwner, + Busy, + PendingRelease, + StaleLease, + Overflow, +} + +#[derive(Clone, Copy)] +struct Slot { + owner: u64, + generation: u64, + pins: u32, + last_use: u64, +} + +impl Slot { + const EMPTY: Self = Self { owner: 0, generation: 0, pins: 0, last_use: 0 }; + + fn lease(self, index: usize) -> Lease { + Lease { slot: FIRST + index as u16, owner: self.owner, generation: self.generation } + } +} + +pub(super) struct State { + slots: [Slot; COUNT], + clock: u64, +} + +impl State { + pub(super) const fn new() -> Self { Self { slots: [Slot::EMPTY; COUNT], clock: 0 } } + + /// Constant-time cached-owner path; a recycled generation cannot be pinned. + pub(super) fn pin(&mut self, lease: Lease) -> Result { + let index = lease.slot.checked_sub(FIRST).ok_or(Error::StaleLease)? as usize; + let slot = self.slots.get_mut(index).ok_or(Error::StaleLease)?; + if slot.owner != lease.owner || slot.generation != lease.generation || slot.owner == 0 { + return Err(Error::StaleLease); + } + let clock = self.clock.checked_add(1).ok_or(Error::Overflow)?; + let pins = slot.pins.checked_add(1).ok_or(Error::Overflow)?; + slot.pins = pins; + slot.last_use = clock; + self.clock = clock; + Ok(lease) + } + + /// A successful claim is a preparation pin. It must remain pinned through + /// any out-of-lock construction and every firmware reference, not just + /// through publication. Existing-owner reacquisition does not reassign it. + pub(super) fn claim(&mut self, owner: u64, mut released: impl FnMut(Lease) -> bool) + -> Result + { + if owner == 0 { return Err(Error::BadOwner); } + let next_clock = self.clock.checked_add(1).ok_or(Error::Overflow)?; + if let Some(index) = self.slots.iter().position(|s| s.owner == owner) { + let slot = &mut self.slots[index]; + let pins = slot.pins.checked_add(1).ok_or(Error::Overflow)?; + slot.pins = pins; + slot.last_use = next_clock; + self.clock = next_clock; + return Ok(Claim { lease: slot.lease(index), displaced: None }); + } + let mut candidate = self.slots.iter().position(|s| s.owner == 0); + let mut displaced = None; + if candidate.is_none() { + // One bounded scan, with no allocation and at most one release + // query per idle slot. Normal cached-owner pinning is O(1). + let mut idle = false; + for (index, slot) in self.slots.iter().enumerate() { + if slot.pins != 0 { continue; } + idle = true; + if candidate.is_some_and(|best| self.slots[best].last_use <= slot.last_use) { continue; } + let old = slot.lease(index); + if released(old) { + candidate = Some(index); + displaced = Some(old); + } + } + if candidate.is_none() { + return Err(if idle { Error::PendingRelease } else { Error::Busy }); + } + } + let index = candidate.ok_or(Error::Busy)?; + let generation = self.slots[index].generation.checked_add(1).ok_or(Error::Overflow)?; + self.slots[index] = Slot { owner, generation, pins: 1, last_use: next_clock }; + self.clock = next_clock; + Ok(Claim { lease: self.slots[index].lease(index), displaced }) + } + + /// An old owner or old generation must never release the successor's pin. + pub(super) fn unpin(&mut self, lease: Lease) -> Result { + let index = lease.slot.checked_sub(FIRST).ok_or(Error::StaleLease)? as usize; + let slot = self.slots.get_mut(index).ok_or(Error::StaleLease)?; + if slot.owner != lease.owner || slot.generation != lease.generation || slot.pins == 0 { + return Err(Error::StaleLease); + } + slot.pins -= 1; + Ok(slot.pins == 0) + } + + /// Pool destruction may also need a pending release to finish. The caller + /// retains its memory when this refuses; dropping it is not a release proof. + pub(super) fn remove(&mut self, owner: u64, released: impl FnOnce(Lease) -> bool) + -> Result<(), Error> + { + if owner == 0 { return Err(Error::BadOwner); } + let Some(index) = self.slots.iter().position(|s| s.owner == owner) else { return Ok(()); }; + let slot = self.slots[index]; + if slot.pins != 0 { return Err(Error::Busy); } + if !released(slot.lease(index)) { return Err(Error::PendingRelease); } + self.slots[index].owner = 0; + Ok(()) + } + + /// Return one cached, unpinned registration only after its release proof. + pub(super) fn reclaim_one(&mut self, mut released: impl FnMut(Lease) -> bool) -> Option { + let index = (0..COUNT) + .filter(|&i| self.slots[i].owner != 0 && self.slots[i].pins == 0) + .filter(|&i| released(self.slots[i].lease(i))) + .min_by_key(|&i| self.slots[i].last_use)?; + let old = self.slots[index].lease(index); + self.slots[index].owner = 0; + Some(old) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn full() -> (State, Vec) { + let mut state = State::new(); + let leases = (1..=COUNT as u64).map(|owner| + state.claim(owner, |_| panic!("free slot must not need release proof")).unwrap().lease + ).collect(); + (state, leases) + } + + #[test] fn bounded_physical_namespace() { + let (state, leases) = full(); + assert_eq!(leases.iter().map(|l| l.slot).collect::>(), (129..=255).collect::>()); + assert_eq!(state.slots.len(), COUNT); + } + + #[test] fn all_preparation_pins_prevent_reassignment() { + let (mut state, _) = full(); + assert_eq!(state.claim(1000, |_| panic!("pinned slot must not be offered")), Err(Error::Busy)); + } + + #[test] fn unpinned_is_not_firmware_released() { + let (mut state, leases) = full(); + for lease in &leases { state.unpin(*lease).unwrap(); } + assert_eq!(state.claim(1000, |_| false), Err(Error::PendingRelease)); + for (i, lease) in leases.iter().enumerate() { assert_eq!(state.slots[i].lease(i), *lease); } + } + + #[test] fn released_slot_can_serve_another_logical_owner() { + let (mut state, leases) = full(); + state.unpin(leases[13]).unwrap(); + let claim = state.claim(1000, |old| old == leases[13]).unwrap(); + assert_eq!(claim.displaced, Some(leases[13])); + assert_eq!(claim.lease.slot, leases[13].slot); + assert!(claim.lease.generation > leases[13].generation); + assert_eq!(state.unpin(leases[13]), Err(Error::StaleLease)); + assert!(state.unpin(claim.lease).unwrap()); + } + + #[test] fn same_owner_reuses_without_release_proof_or_generation_change() { + let mut state = State::new(); + let a = state.claim(1, |_| false).unwrap().lease; + let b = state.claim(1, |_| panic!("same owner is not reassigned")).unwrap().lease; + assert_eq!(a, b); + assert!(!state.unpin(a).unwrap()); + assert!(state.unpin(b).unwrap()); + assert_eq!(state.unpin(a), Err(Error::StaleLease)); + } + + #[test] fn stale_generation_cannot_unpin_returned_owner() { + let (mut state, leases) = full(); + state.unpin(leases[0]).unwrap(); + let other = state.claim(1000, |_| true).unwrap().lease; + state.unpin(other).unwrap(); + let returned = state.claim(leases[0].owner, |_| true).unwrap().lease; + assert_eq!(returned.slot, leases[0].slot); + assert_eq!(state.unpin(leases[0]), Err(Error::StaleLease)); + assert!(state.unpin(returned).unwrap()); + } + + #[test] fn thousands_of_logical_owners_do_not_expand_the_namespace() { + let mut state = State::new(); + for owner in 1..=4096 { + let lease = state.claim(owner, |_| true).unwrap().lease; + assert!((129..=255).contains(&lease.slot)); + assert!(state.unpin(lease).unwrap()); + } + assert_eq!(state.slots.iter().filter(|s| s.owner != 0).count(), COUNT); + } + + #[test] fn removal_waits_for_both_pins_and_firmware() { + let mut state = State::new(); + let lease = state.claim(1, |_| false).unwrap().lease; + assert_eq!(state.remove(1, |_| true), Err(Error::Busy)); + state.unpin(lease).unwrap(); + assert_eq!(state.remove(1, |_| false), Err(Error::PendingRelease)); + state.remove(1, |old| old == lease).unwrap(); + assert_eq!(state.unpin(lease), Err(Error::StaleLease)); + } + + #[test] fn cached_pin_preserves_lease_and_blocks_reclaim() { + let (mut state, leases) = full(); + state.unpin(leases[0]).unwrap(); + assert_eq!(state.pin(leases[0]), Ok(leases[0])); + assert_eq!(state.claim(1000, |_| true), Err(Error::Busy)); + } + + #[test] fn stale_cached_pin_cannot_name_new_owner() { + let (mut state, leases) = full(); + state.unpin(leases[0]).unwrap(); + let replacement = state.claim(1000, |_| true).unwrap().lease; + assert_eq!(state.pin(leases[0]), Err(Error::StaleLease)); + assert!(state.unpin(replacement).unwrap()); + } +} diff --git a/drivers/gpu/drm/asahi/g17/fw/channels.rs b/drivers/gpu/drm/asahi/g17/fw/channels.rs index 26c6adcf34c83..2d145f4511a2d 100644 --- a/drivers/gpu/drm/asahi/g17/fw/channels.rs +++ b/drivers/gpu/drm/asahi/g17/fw/channels.rs @@ -166,10 +166,10 @@ impl EventRecord { } /// Type 7 is a retirement witness only on the firmware whose layout and - /// visibility semantics are qualified. 25G83 must keep consuming it as an - /// unknown event until its own fault contract is established. - pub(crate) fn decode_for(&self, graph: super::variant::InitGraph) -> Result { - if graph == super::variant::InitGraph::V25G83 && self.tag() == Self::COMPUTE_FAULT { + /// visibility semantics are qualified. 25G83 and M5 must keep consuming it as an + /// unknown event until their own fault contract is established. + pub(crate) fn decode_for(&self, graph: super::variant::InitGraph, m5: bool) -> Result { + if (m5 || graph == super::variant::InitGraph::V25G83) && self.tag() == Self::COMPUTE_FAULT { return Ok(Event::Unknown(Self::COMPUTE_FAULT)); } self.decode() @@ -273,7 +273,9 @@ impl ControlRecord { match self { Self::TvbGrow(_) => DeviceControlOp::TvbGrowReply, Self::IdlePowerOff(_) => DeviceControlOp::IdlePowerOff, - Self::SchedulerStateRelease(_) => DeviceControlOp::SchedulerStateRelease, + Self::SchedulerStateRelease(record) => if record.opcode == DeviceControlOp::M5SchedulerStateRelease as u32 { + DeviceControlOp::M5SchedulerStateRelease + } else { DeviceControlOp::SchedulerStateRelease }, Self::FreeListRelease(_) => DeviceControlOp::FreeListRelease, Self::ContextKill(_) => DeviceControlOp::ContextKill, } @@ -305,6 +307,8 @@ pub(crate) enum DeviceControlOp { IdlePowerOff = 0x0a, /// Release a logical queue's scheduler state ([`SchedulerStateRelease`]). SchedulerStateRelease = 0x14, + /// M5 encoding of the same scheduler-state release. + M5SchedulerStateRelease = 0x13, /// Release a free list's buffer slot ([`FreeListRelease`]). FreeListRelease = 0x2e, /// Stop the queues named by a closed logical scheduler. @@ -574,10 +578,11 @@ mod tests { fn v25g83_does_not_accept_an_unqualified_fault_witness() { use super::super::variant::InitGraph; let fault = record(&[7, 1, 2, 0x39, 0x300]); - assert_eq!(fault.decode_for(InitGraph::G17P), + assert_eq!(fault.decode_for(InitGraph::G17P, false), Ok(Event::ComputeFault { qid: 0x39, stamp: 0x300 })); - assert_eq!(fault.decode_for(InitGraph::V25G83), Ok(Event::Unknown(7))); - assert_eq!(record(&[7, 1, 2, 128, 0x301]).decode_for(InitGraph::V25G83), + assert_eq!(fault.decode_for(InitGraph::G17P, true), Ok(Event::Unknown(7))); + assert_eq!(fault.decode_for(InitGraph::V25G83, false), Ok(Event::Unknown(7))); + assert_eq!(record(&[7, 1, 2, 128, 0x301]).decode_for(InitGraph::V25G83, false), Ok(Event::Unknown(7))); } diff --git a/drivers/gpu/drm/asahi/g17/fw/compute.rs b/drivers/gpu/drm/asahi/g17/fw/compute.rs index 91b3a2f58334c..76f4f96bd6ea6 100644 --- a/drivers/gpu/drm/asahi/g17/fw/compute.rs +++ b/drivers/gpu/drm/asahi/g17/fw/compute.rs @@ -180,6 +180,16 @@ pub(crate) struct OperandState { } impl OperandState { + /// Size of one preemption bank, the region the M5 program exchanges. + pub(crate) const fn bank_bytes(&self) -> u32 { + self.r0a5c9 as u32 + } + + /// Offset of the per-cluster tails, the M5 program's auxiliary area. + pub(crate) const fn tail_offset(&self) -> u32 { + self.r140a8 as u32 + } + /// One cluster of six cores (A18 Pro). pub(crate) const ONE_BY_SIX: OperandState = OperandState { r10229: 0x12800, @@ -209,6 +219,17 @@ impl OperandState { r0a5c9: 0x3c000, size: 0x80000, }; + /// Eight clusters of ten cores (M5 Ultra): bank stride 0x78400, tails 0x4000. The size is + /// the M5 compute queues' operand buffer, the tails' end rounded up to 16 KiB. + pub(crate) const EIGHT_BY_TEN: OperandState = OperandState { + r10229: 0xf0800, + r140a8: 0xf4800, + r10099: 0x78405, + r10091: 0xf0400, + r0a5c1: 0x0005, + r0a5c9: 0x78000, + size: 0xf8000, + }; } /// Checkpoints into the main register array, as entry counts: after the diff --git a/drivers/gpu/drm/asahi/g17/fw/freelist.rs b/drivers/gpu/drm/asahi/g17/fw/freelist.rs index 86fdc5db75a8f..56c1f2324e2a1 100644 --- a/drivers/gpu/drm/asahi/g17/fw/freelist.rs +++ b/drivers/gpu/drm/asahi/g17/fw/freelist.rs @@ -13,7 +13,7 @@ pub(crate) const RENDER_BLOCKS: usize = 44; pub(crate) const COMPUTE_BLOCKS: usize = 21; /// Render reserves eight run words per block; compute packs runs consecutively. -#[derive(Copy, Clone, Debug)] +#[derive(Copy, Clone, Debug, PartialEq, Eq)] pub(crate) enum RunLayout { Render, Compute, diff --git a/drivers/gpu/drm/asahi/g17/fw/kick.rs b/drivers/gpu/drm/asahi/g17/fw/kick.rs index 2a035524bdffb..8dd18ac550a46 100644 --- a/drivers/gpu/drm/asahi/g17/fw/kick.rs +++ b/drivers/gpu/drm/asahi/g17/fw/kick.rs @@ -369,8 +369,16 @@ pub(crate) struct KickArgs<'a> { /// queue. A queue's first kick names `timestamp.prev()` on render queues /// and [`KickTimestamp::ZERO`] on compute queues. pub(crate) parent: KickTimestamp, + /// The parent's own predecessor on the same queue, a further implicit dependency after the + /// parent: with the G17D forms a fragment kick names it while its queue remembers a previous + /// kick ([`super::super::kick::Queue::ancestor`]); `None` elsewhere. + pub(crate) ancestor: Option, /// Kicks of other queues this kick must wait for. pub(crate) barriers: &'a [KickDependency], + /// Two-bit completion classes for each explicit dependency followed by + /// the implicit parent and the ancestor if any. Interfaces without typed + /// dependencies use `None`. + pub(crate) dependency_types: Option<&'a [u8]>, /// GPU address of the work descriptor (firmware alias). pub(crate) descriptor_va: u64, /// GPU address of the queue record. @@ -383,12 +391,23 @@ pub(crate) struct KickArgs<'a> { pub(crate) event_mask: [u64; 4], /// Register arrays bound to the kick. pub(crate) register_arrays: [Option; 4], - /// The compute register program includes a shader-context scratch request. + /// The stage's register program includes a shader-context scratch (SCS) + /// bracket (header bit 61; compute, TA and 3D alike). pub(crate) scs_enabled: bool, + /// The stage's register program includes a matrix bracket (bit 63). + pub(crate) compute_matrix: bool, /// Priority class of the queue. pub(crate) priority: u8, } +/// Per-stage (TA, 3D) kick header flags of a render, derived from the same +/// state that emits the SCS and matrix register brackets. +#[derive(Copy, Clone, Debug, Default, PartialEq, Eq)] +pub(crate) struct StageKickFlags { + pub(crate) scratch: [bool; 2], + pub(crate) matrix: [bool; 2], +} + /// One kick ring entry. /// /// Live state: none. The host writes the whole entry before announcing it and @@ -397,7 +416,8 @@ pub(crate) struct KickArgs<'a> { #[derive(Copy, Clone, Debug)] pub(crate) struct KickEntry { /// Timestamp in bits 41:2, QID in bits 48:42, MCache table present in bit - /// 58, queue priority in bits 60:59 and compute scratch enabled in bit 61. + /// 58, queue priority in bits 60:59, SCS bracket in the stage's register + /// program in bit 61 and matrix bracket in bit 63. pub(crate) header: u64, /// MCache table: address >> 5 in bits 43:6, count - 1 in bits 59:54. pub(crate) mcache: u64, @@ -418,7 +438,10 @@ pub(crate) struct KickEntry { pub(crate) register_arrays: [u64; 4], pub(crate) unk_170: u64, pub(crate) unk_178: u64, - pub(crate) pad_180: [u8; KICK_ENTRY_STRIDE - 0x180], + /// Two bits per valid dependency, in the same order as `dependencies`. + /// Zero on interfaces without typed dependencies. + pub(crate) dependency_types: u64, + pub(crate) pad_188: [u8; KICK_ENTRY_STRIDE - 0x188], } static_assert!(size_of::() == KICK_ENTRY_STRIDE); @@ -426,6 +449,7 @@ static_assert!(core::mem::offset_of!(KickEntry, dependency_control) == 0x20); static_assert!(core::mem::offset_of!(KickEntry, event_mask) == 0x128); static_assert!(core::mem::offset_of!(KickEntry, register_arrays) == 0x150); static_assert!(core::mem::offset_of!(KickEntry, unk_178) == 0x178); +static_assert!(core::mem::offset_of!(KickEntry, dependency_types) == 0x180); static_assert!(size_of::() == 0xc); static_assert!(size_of::() == 0x10); static_assert!(size_of::() == 0x720); @@ -438,6 +462,7 @@ impl KickEntry { const MCACHE_PRESENT: u64 = 1 << 58; const PRIORITY_SHIFT: u32 = 59; const SCS_ENABLED: u64 = 1 << 61; + const COMPUTE_MATRIX: u64 = 1 << 63; const QOS_SLOT_SHIFT: u32 = 32; const QOS_CLASS_SHIFT: u32 = 40; const QOS_CLASS_MAX: u8 = 0x1f; @@ -449,8 +474,10 @@ impl KickEntry { /// Builds the entry described by `args`. pub(crate) fn new(args: &KickArgs<'_>) -> Result { let barriers = args.barriers.len(); + let count = barriers + 1 + usize::from(args.ancestor.is_some()); if args.qid > QID_MAX || barriers > KICK_BARRIERS_MAX + || count > KICK_DEPENDENCIES || args.qos.class > Self::QOS_CLASS_MAX || args.priority >= Policy::PRIORITIES { @@ -463,6 +490,9 @@ impl KickEntry { if args.scs_enabled { header |= Self::SCS_ENABLED; } + if args.compute_matrix { + header |= Self::COMPUTE_MATRIX; + } let mcache = match args.mcache { Some(table) => { header |= Self::MCACHE_PRESENT; @@ -475,12 +505,21 @@ impl KickEntry { for (slot, barrier) in dependencies.iter_mut().zip(args.barriers) { *slot = barrier.word(); } - dependencies[barriers] = KickDependency { - qid: args.qid, - timestamp: args.parent, + // The implicit dependencies on this queue follow the barriers: the parent, then the + // ancestor if any (`count` fits the array). + let implicit = [Some(args.parent), args.ancestor].into_iter().flatten(); + for (slot, timestamp) in dependencies[barriers..].iter_mut().zip(implicit) { + *slot = KickDependency { qid: args.qid, timestamp }.word(); + } + let mut dependency_types = 0; + if let Some(types) = args.dependency_types { + if types.len() != count || types.iter().any(|kind| *kind > 3) { + return Err(EINVAL); + } + for (index, kind) in types.iter().enumerate() { + dependency_types |= u64::from(*kind) << (2 * index); + } } - .word(); - let count = barriers + 1; let mask = if count == KICK_DEPENDENCIES { u32::MAX } else { @@ -502,7 +541,8 @@ impl KickEntry { register_arrays: args.register_arrays.map(RegisterArrayBinding::word), unk_170: 0, unk_178: Self::UNK_178, - pad_180: [0; KICK_ENTRY_STRIDE - 0x180], + dependency_types, + pad_188: [0; KICK_ENTRY_STRIDE - 0x188], }) } } diff --git a/drivers/gpu/drm/asahi/g17/fw/m5/compute.rs b/drivers/gpu/drm/asahi/g17/fw/m5/compute.rs new file mode 100644 index 0000000000000..1d969ffa42e8f --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fw/m5/compute.rs @@ -0,0 +1,549 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! M5 compute register programs. The operand and preemption layouts differ +//! from the other G17 profile; sharing a register count does not imply a shared +//! register program. M5 scratch has its own layout and two control writes. + +use kernel::prelude::*; +use super::super::{kick::RegisterWrite, offset_va, work_key}; +use super::initdata::Board; + +pub(crate) const MAIN_COUNT: usize = 40; +pub(crate) const ALTERNATE_COUNT: usize = 4; +pub(crate) const PREEMPT_BYTES: usize = 0x4000; +pub(crate) const CHECKPOINTS: [u16; 4] = [37, 39, 39, 40]; + +/// A dispatch control word for `board`: above four clusters it carries the cluster-topology term, +/// `0x9c` in the form for the firmware feature the 27.0 profile leaves clear (`0x1c` is the +/// feature-set form). The term is an OR. Compute and render dispatch words both take it. +pub(crate) fn dispatch_control(board: &Board, base: u64) -> u64 { + topology_dispatch_control(base, board.clusters) +} + +/// [`dispatch_control`] for `clusters`. +pub(crate) const fn topology_dispatch_control(base: u64, clusters: u32) -> u64 { + if clusters > 4 { base | 0x9c } else { base } +} + +/// Dependency class of a pipelined compute dependency: the class the kick-queue encoder supplies +/// for compute work (tiling's is also 2, fragment's 3). +pub(crate) const COMPUTE_PIPELINED_DEPENDENCY: u8 = 2; + +/// The completion class, which holds a kick until the kick it names has completed. +pub(crate) const COMPLETION_DEPENDENCY: u8 = 1; + +/// Dependency class of an explicit dependency of a compute kick on `board`. +pub(crate) fn explicit_dependency(board: &Board) -> u8 { + if board.completion_inputs() { COMPLETION_DEPENDENCY } else { COMPUTE_PIPELINED_DEPENDENCY } +} + +/// Dependency class of a compute kick's implicit parent (the previous kick of its queue) on +/// `board`. With the counted forms ([`counted_forms`]) it is class 1, which holds the kick until +/// its parent has completed. On boards whose explicit waits take the completion class +/// ([`Board::completion_inputs`]) a parent the host has already retired (`parent_retired`) takes +/// the pipelined class instead, as macOS's late-published M5 Pro kicks do: there is nothing left +/// to hold for. Otherwise it is the pipelined class, with which a kick may start while its parent +/// still runs. +pub(crate) fn parent_dependency(board: &Board, parent_retired: bool) -> u8 { + if !counted_forms(board) || (parent_retired && board.completion_inputs()) { + COMPUTE_PIPELINED_DEPENDENCY + } else { + COMPLETION_DEPENDENCY + } +} + +/// Whether compute kicks on `board` count their completion ordinal and hold for their parent: +/// with the G17D forms, and on boards whose explicit waits take the completion class +/// ([`Board::completion_inputs`]), as macOS encodes M5 Pro compute. +pub(crate) fn counted_forms(board: &Board) -> bool { + board.g17d_forms() || board.completion_inputs() +} + +/// The completion ordinal of a compute command on `board`. With the counted forms +/// ([`counted_forms`]) it is the count of compute commands its hardware queue published before it +/// (zero for a new queue), wrapping as a u32; render commands count separately. Otherwise it is 0 +/// on every kick. +pub(crate) fn completion_ordinal(board: &Board, published_before: u32) -> u32 { + if counted_forms(board) { published_before } else { 0 } +} + +/// Contexts per engine in the event space: a pool's event is 64 x its engine + its context. +const POOL_EVENT_CONTEXTS: u32 = 64; +/// The engine of a compute pool in the event space: the firmware's record of each compute pool +/// names engine 2 beside the pool's context. +const POOL_EVENT_ENGINE: u32 = 2; +/// The entry-signal gate: event 254, which the firmware clears in the kick a current-kick +/// notification names once it has handled that record. +const ENTRY_GATE_EVENT: u32 = 254; + +/// The event mask of a compute kick of context `context_id` on `board`, published with its +/// current-kick notification. With the G17D forms ([`Board::g17d_forms`]) the kick waits on its +/// own context's pool event (64 x 2 + the context), which the firmware holds while it processes +/// that pool's dependencies, and on the entry-signal gate; a context of 64 or more is EINVAL. With +/// the other counted forms ([`counted_forms`]) it is the pool event alone. Otherwise it is the +/// fixed `[0, 0, 8, 0]` (event 131, context 3's pool event, without the gate). +pub(crate) fn kick_event_mask(board: &Board, context_id: u32) -> Result<[u64; 4]> { + if !counted_forms(board) { + return Ok([0, 0, 8, 0]); + } + if context_id >= POOL_EVENT_CONTEXTS { + return Err(EINVAL); + } + let mut mask = [0u64; 4]; + let pool = POOL_EVENT_ENGINE * POOL_EVENT_CONTEXTS + context_id; + let events = [Some(pool), board.g17d_forms().then_some(ENTRY_GATE_EVENT)]; + for event in events.into_iter().flatten() { + mask[(event / 64) as usize] |= 1 << (event % 64); + } + Ok(mask) +} + +/// The descriptor words of a DRM compute kick that follow the part, not the command: the +/// preemption-state offset, the dispatch control base (before [`dispatch_control`]'s topology +/// term) and the auxiliary control's first write. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct KickTuple { + pub(crate) preempt_state_offset: u32, + pub(crate) dispatch_base: u64, + pub(crate) auxiliary_control: u64, +} + +/// The tuple of J714 compute. +pub(crate) const NORMAL_TUPLE: KickTuple = + KickTuple { preempt_state_offset: 0x1b80, dispatch_base: 0x14c0_8a60, auxiliary_control: 1 }; +/// The tuple the arithmetic qualification runs with on every part. The G17D runs DRM compute with +/// it: with [`NORMAL_TUPLE`] its kicks never complete. +pub(crate) const QUALIFIED_TUPLE: KickTuple = + KickTuple { preempt_state_offset: 0x1480, dispatch_base: 0x10c0_8860, auxiliary_control: 0 }; + +/// The DRM compute tuple of `board` ([`Board::g17d_forms`]) for a kick whose parent takes class +/// `parent_class`. Where explicit waits take the completion class ([`Board::completion_inputs`]) +/// it is macOS's M5 Pro form: the J714 preemption offset with the qualified dispatch base, and the +/// auxiliary control set only when the kick's parent is pipelined. +pub(crate) fn kick_tuple(board: &Board, parent_class: u8) -> KickTuple { + if board.g17d_forms() { + QUALIFIED_TUPLE + } else if board.completion_inputs() { + KickTuple { + preempt_state_offset: NORMAL_TUPLE.preempt_state_offset, + dispatch_base: QUALIFIED_TUPLE.dispatch_base, + auxiliary_control: u64::from(parent_class == COMPUTE_PIPELINED_DEPENDENCY), + } + } else { + NORMAL_TUPLE + } +} + +const _: () = { + assert!(topology_dispatch_control(0x14c0_8a60, 8) == 0x14c0_8afc); + assert!(topology_dispatch_control(0x10c0_8860, 8) == 0x10c0_88fc); + assert!(topology_dispatch_control(0x14c0_8a60, 4) == 0x14c0_8a60); +}; + +/// The SCS scratch allocation. Pages are 4 KiB USC pool pages, independent +/// of the host's CPU page size. Layout sections and control registers are +/// specific to M5; the other G17 profile uses a different fixed region. +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) struct ScratchRequest { + layout: u64, + driver_end: u32, + pages: u16, +} + +impl ScratchRequest { + pub(crate) const NONE: Self = Self { layout: 0, driver_end: 0, pages: 0 }; + + pub(crate) fn new(layout: u64) -> Result { + let cliques = (layout >> 24) & 0xff; + if cliques == 0 { return Ok(Self::NONE); } + let depth = layout & 0xffff; + let levels = (layout >> 16) & 0xff; + let user = (layout >> 32) & 0xffff; + let driver = layout >> 48; + let threads = cliques * 32; + // Four fixed sections followed by per-thread stacks, driver/user + // payloads and per-level state. All inputs are bounded bitfields. + let driver_end = 0x68000 + threads * (depth * 8 + driver); + // The register naming the end of the driver section holds a 24-bit + // byte offset. Reject truncation even if the total page count fits. + if driver_end > 0xffffff { return Err(EINVAL); } + let bytes = driver_end + threads * user + + cliques * 256 * (levels + u64::from(levels > 1)); + let pages = (bytes + 0x1fff) >> 12; + Ok(Self { layout, driver_end: driver_end as u32, + pages: u16::try_from(pages).map_err(|_| EINVAL)? }) + } + + pub(crate) fn enabled(self) -> bool { self.pages != 0 } + pub(crate) fn layout(self) -> u64 { self.layout } + pub(crate) fn pages(self) -> u16 { self.pages } + pub(crate) fn size_word(self) -> u64 { + let pages = u64::from(self.pages); + pages | (pages.div_ceil(256) << 16) + } + pub(crate) fn window_word(self) -> u64 { + if self.enabled() { (1 << 24) | (((self.layout >> 24) & 0xff) << 16) } + else { 0xff0000 } + } + pub(crate) fn stack_word(self) -> u64 { + if self.enabled() { self.layout & 0xfff8 } else { 8 } + } + pub(crate) fn driver_word(self) -> u64 { + if !self.enabled() { return 0; } + let levels = (self.layout >> 16) & 0xff; + u64::from(self.driver_end) | ((levels + u64::from(levels > 1)) << 24) + } + fn checkpoints(self) -> [u16; 4] { + CHECKPOINTS.map(|count| count + if self.enabled() { 2 } else { 0 }) + } +} + +/// Per-command intersection table bindings. The pool contains 128-byte +/// blocks of eight 16-byte program entries. Addresses are UAT user addresses; +/// ownership and readable ranges are checked by command validation. +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) struct IntersectionTables { + pub(crate) pool: u64, + pub(crate) pool_bytes: u64, + first: u32, + count: u32, + pub(crate) default_program: u64, + default_control: u64, +} + +impl IntersectionTables { + pub(crate) const NONE: Self = Self { + pool: 0, pool_bytes: 0, first: 0, count: 0, + default_program: 0, default_control: 0, + }; + + pub(crate) fn new(pool: u64, pool_bytes: u64, first: u32, count: u32, + default_program: u64, default_control: u64) -> Result { + if pool == 0 { + if pool_bytes != 0 || first != 0 || count != 0 + || default_program != 0 || default_control != 0 { + return Err(EINVAL); + } + return Ok(Self::NONE); + } + // Resource sizes are encoded in 16 KiB pages. Table lengths + // have 18 block-count bits; truncating either field changes addressing. + let end = pool.checked_add(pool_bytes).ok_or(EINVAL)?; + let blocks_end = u64::from(first) + u64::from(count); + let code = default_program & 0x0000_ffff_ffff_ffff; + if pool & 0x3fff != 0 || pool_bytes == 0 || pool_bytes & 0x3fff != 0 + || end > (1u64 << 48) || (pool_bytes >> 14) >= (1u64 << 25) + || count == 0 || count >= (1 << 18) || blocks_end * 128 > pool_bytes + || code & 63 != 0 + || default_control & !0x0000_7fff_0000_3fff != 0 + || (code == 0 && (default_program != 0 || default_control != 0)) + || (code != 0 && default_control & (1u64 << 40) == 0) { + return Err(EINVAL); + } + Ok(Self { pool, pool_bytes, first, count, default_program, default_control }) + } + + pub(crate) fn enabled(self) -> bool { self.pool != 0 } + + pub(crate) fn words(self) -> [u64; 6] { + if !self.enabled() { return [0; 6]; } + [self.pool + u64::from(self.first) * 128 | (1u64 << 51), + 0xc0000000 | (u64::from(self.count) << 7), self.pool, + self.pool_bytes >> 14, self.default_program, self.default_control] + } +} + +/// Addresses and policy owned by a queued compute command. +pub(crate) struct RegisterArgs { + pub(crate) matrix_ops: bool, + pub(crate) scratch: ScratchRequest, + pub(crate) intersection: IntersectionTables, + /// Userspace's opaque auxiliary allocation word, independent of the SCS layout. + pub(crate) auxiliary_allocation: u32, + pub(crate) preempt_va: u64, + pub(crate) cdm_va: u64, + pub(crate) operand_va: u64, + pub(crate) usage_va: u64, + /// Private-memory pool slot, as named by the usage record. + pub(crate) free_list_slot: u32, + pub(crate) context_id: u16, + pub(crate) work_state: u32, + /// Packed predecessor/current object identifiers. + pub(crate) object_ids: u64, + /// Pool target and resource policy for the program's first resource-policy write. + pub(crate) resource_policy: u64, + /// Start of the four eight-byte state words in the preemption buffer. + pub(crate) preempt_state_offset: u32, + /// Size of one operand-state region; the program exchanges two regions. + pub(crate) operand_region_bytes: u32, + pub(crate) operand_aux_offset: u32, + /// Size of the operand-state buffer at `operand_va`. + pub(crate) operand_bytes: usize, + pub(crate) resource_control: u64, + pub(crate) dispatch_control: u64, + pub(crate) auxiliary_control: u64, +} + +impl RegisterArgs { + fn validate(&self) -> Result { + if self.preempt_va & 0x3fff != 0 || self.operand_va & 0x3fff != 0 + || self.cdm_va & 7 != 0 || self.usage_va & 7 != 0 || self.context_id >= 64 { + return Err(EINVAL); + } + offset_va(self.preempt_va, PREEMPT_BYTES as u64)?; + offset_va(self.operand_va, self.operand_bytes as u64)?; + if self.preempt_state_offset & 7 != 0 + || u64::from(self.preempt_state_offset) + 32 > PREEMPT_BYTES as u64 + || self.operand_region_bytes == 0 || self.operand_region_bytes & 0xfff != 0 + || u64::from(self.operand_region_bytes) * 2 + 0x808 > self.operand_bytes as u64 + || self.operand_aux_offset & 7 != 0 + || u64::from(self.operand_aux_offset) + 8 > self.operand_bytes as u64 { + return Err(EINVAL); + } + Ok(()) + } +} + +/// Construct the complete primary program before publishing its descriptor. +pub(crate) fn main(args: &RegisterArgs) -> Result<[RegisterWrite; MAIN_COUNT]> { + args.validate()?; + let preempt = args.preempt_va; + let operand = args.operand_va; + let ids = args.object_ids; + let intersection = args.intersection.words(); + let state = u64::from(args.preempt_state_offset); + let region = u64::from(args.operand_region_bytes); + let key = work_key(args.context_id, args.work_state); + Ok([ + RegisterWrite::new(0x1a510, preempt), + RegisterWrite::new(0x1a420, args.cdm_va), + RegisterWrite::new(0x1a4d0, offset_va(preempt, state)?), + RegisterWrite::new(0x1a4d8, offset_va(preempt, state + 8)?), + RegisterWrite::new(0x1a4e0, offset_va(preempt, state + 16)?), + RegisterWrite::new(0x1a4e8, offset_va(preempt, state + 24)?), + RegisterWrite::new(0x1a440, 0x154024201), + RegisterWrite::new(0x0d4a1, args.resource_control), + RegisterWrite::new(0x0d411, 0x200000001), + RegisterWrite::new(0x0a599, args.resource_policy), + RegisterWrite::new(0x1a458, args.dispatch_control), + RegisterWrite::new(0x101d9, 0x1c), + RegisterWrite::new(0x1a089, intersection[0]), + RegisterWrite::new(0x1a091, intersection[1]), + RegisterWrite::new(0x1a059, intersection[2]), + RegisterWrite::new(0x1a061, intersection[3]), + RegisterWrite::new(0x1a0b9, intersection[4]), + RegisterWrite::new(0x1a0c1, intersection[5]), + RegisterWrite::new(0x101d1, args.scratch.size_word()), + RegisterWrite::new(0x0d479, u64::from(args.auxiliary_allocation)), + RegisterWrite::new(0x1a0e9, args.scratch.stack_word()), + RegisterWrite::new(0x1a141, args.scratch.driver_word()), + RegisterWrite::new(0x107a1, args.scratch.window_word() | (u64::from(args.matrix_ops) << 25)), + RegisterWrite::new(0x1a540, ids), + RegisterWrite::new(0x014a9, ids), + RegisterWrite::new(0x0a351, ids), + RegisterWrite::new(0x0a731, args.auxiliary_control), + RegisterWrite::new(0x10201, key), + RegisterWrite::new(0x10428, key), + RegisterWrite::new(0x14028, args.free_list_slot.into()), + RegisterWrite::new(0x14070, args.usage_va | 1), + RegisterWrite::new(0x10229, offset_va(operand, region * 2 + 0x800)?), + RegisterWrite::new(0x140a8, offset_va(operand, args.operand_aux_offset.into())?), + RegisterWrite::new(0x10099, offset_va(operand, region + 0x405)?), + RegisterWrite::new(0x10091, offset_va(operand, region * 2 + 0x400)?), + RegisterWrite::new(0x0a5c1, offset_va(operand, 5)?), + RegisterWrite::new(0x0a5c9, offset_va(operand, region)?), + RegisterWrite::new(0x0a731, 0), + RegisterWrite::new(0x1a440, 0x154024209), + RegisterWrite::new(0x0a599, 0x6000400020), +]) +} + +fn program(args: &RegisterArgs) -> Result<([RegisterWrite; MAIN_COUNT + 4], usize)> { + let main = main(args)?; + let mut writes = [RegisterWrite::new(0, 0); MAIN_COUNT + 4]; + let mut count = 0; + if args.scratch.enabled() { + writes[count] = RegisterWrite::new(0x17e1, 1); + count += 1; + } + if args.matrix_ops { + writes[count] = RegisterWrite::new(0x17f1, 1); + count += 1; + } + for (index, register) in main.iter().enumerate() { + if index == 33 { + if args.scratch.enabled() { + writes[count] = RegisterWrite::new(0x17d9, 1); + count += 1; + } + if args.matrix_ops { + writes[count] = RegisterWrite::new(0x17e9, 1); + count += 1; + } + } + writes[count] = *register; + count += 1; + } + Ok((writes, count)) +} + +/// Operand-state exchange used by the alternate program. +pub(crate) fn alternate(args: &RegisterArgs) -> Result<[RegisterWrite; ALTERNATE_COUNT]> { + args.validate()?; + let operand = args.operand_va; + let region = u64::from(args.operand_region_bytes); + Ok([ + RegisterWrite::new(0x10099, offset_va(operand, 5)?), + RegisterWrite::new(0x10091, offset_va(operand, region)?), + RegisterWrite::new(0x0a5c1, offset_va(operand, region + 0x405)?), + RegisterWrite::new(0x0a5c9, offset_va(operand, region * 2 + 0x400)?), + ]) +} + +/// Register-array bindings in a compute kick. The first checkpoint follows +/// the initial operand-region write, before the second auxiliary-control write. +pub(crate) fn bindings(descriptor_va: u64, scratch: ScratchRequest, matrix_ops: bool) + -> Result<[Option; 4]> +{ + use super::super::kick::RegisterArrayBinding; + Ok([ + Some(RegisterArrayBinding::new(offset_va(descriptor_va, 0x40)?, (scratch.checkpoints()[0] + if matrix_ops { 2 } else { 0 }) as u8)?), + Some(RegisterArrayBinding::new(offset_va(descriptor_va, 0x760)?, ALTERNATE_COUNT as u8)?), + None, + None, + ]) +} + +/// The fixed prefix of a compute descriptor. Storage beyond this prefix is +/// owned and initialized by the runtime allocation, not described here. +pub(crate) const DESCRIPTOR_PREFIX_BYTES: usize = 0x1000; + +/// Per-command fields outside the two register programs. Fields whose bit +/// meanings are not established remain explicit opaque values. +pub(crate) struct DescriptorArgs { + pub(crate) registers: RegisterArgs, + pub(crate) descriptor_va: u64, + pub(crate) submit_sequence: u64, + pub(crate) work_state_va: u64, + pub(crate) cdm_end_va: u64, + pub(crate) stamp_va: [u64; 2], + pub(crate) stamp_value: u32, + pub(crate) qid: u32, + pub(crate) kick_timestamp: u64, + pub(crate) completion_ordinal: u32, + pub(crate) status_va: [u64; 2], + pub(crate) timestamp_va: [u64; 2], + pub(crate) shared_control_va: u64, + pub(crate) event_metadata: [u8; 16], + pub(crate) usage_fw_va: u64, + pub(crate) completion_identity: u64, +} + +/// The M5 descriptor has a shorter reserved region before shared-control +/// state. Register-array offsets are shared with G17, tail offsets are not. +#[repr(C, packed)] +pub(crate) struct Descriptor { + tag: u32, + submit_sequence: u64, + context_id: u32, + work_state_va: u64, + checkpoints: [u16; 4], + reserved_20: [u8; 0x20], + registers: super::super::kick::RegisterArray, + alternate: super::super::kick::RegisterArray, + reserved_e80: [u8; 0x58], + preempt_va: u64, + cdm_end_va: u64, + reserved_ee8: [u8; 0x20], + control: u64, + reserved_f10: [u8; 0x10], + predecessor_id: u32, + reserved_f24: u32, + unk_f28: u32, + firmware_timestamps: [u64; 2], + reserved_f3c: u32, + stamp_va: [u64; 2], + stamp_value: u32, + qid: u32, + reserved_f58: u64, + kick_timestamp: u64, + current_id: u64, + completion_ordinal: u32, + reserved_f74: [u8; 8], + status_va: [u64; 2], + timestamp_va: [u64; 2], + reserved_f9c: [u8; 0x10], + shared_control_va: u64, + shared_control_enabled: u8, + event_metadata: [u8; 16], + usage_fw_va: u64, + completion_identity: u64, + reserved_fd5: [u8; 0x2b], +} + +static_assert!(size_of::() == DESCRIPTOR_PREFIX_BYTES); +static_assert!(core::mem::offset_of!(Descriptor, registers) == 0x40); +static_assert!(core::mem::offset_of!(Descriptor, alternate) == 0x760); +static_assert!(core::mem::offset_of!(Descriptor, stamp_va) == 0xf40); +static_assert!(core::mem::offset_of!(Descriptor, completion_ordinal) == 0xf70); +static_assert!(core::mem::offset_of!(Descriptor, shared_control_va) == 0xfac); +static_assert!(core::mem::offset_of!(Descriptor, usage_fw_va) == 0xfc5); +static_assert!(core::mem::offset_of!(Descriptor, completion_identity) == 0xfcd); + +// SAFETY: Every field consists only of integer values and their arrays. +unsafe impl Zeroable for Descriptor {} + +impl Descriptor { + /// Fill the entire prefix; no address or register becomes visible before + /// validation completes. The owner publishes only after this returns. + pub(crate) fn write(&mut self, args: &DescriptorArgs) -> Result { + self.populate::(args) + } + + /// Populate a prefix already cleared by `KernelObject::initialize`. + /// The caller must provide all-zero storage so reserved bytes stay zero. + pub(crate) fn write_zeroed(&mut self, args: &DescriptorArgs) -> Result { + self.populate::(args) + } + + fn populate(&mut self, args: &DescriptorArgs) -> Result { + if args.cdm_end_va < args.registers.cdm_va || args.qid >= 128 { + return Err(EINVAL); + } + let (main, count) = program(&args.registers)?; + let alternate = alternate(&args.registers)?; + let main_va = offset_va(args.descriptor_va, 0x40)?; + let alternate_va = offset_va(args.descriptor_va, 0x760)?; + // Also validate the kick's encodable addresses before mutating storage. + bindings(args.descriptor_va, args.registers.scratch, args.registers.matrix_ops)?; + if CLEAR { + super::super::clear(self); + } + self.tag = 3; + self.submit_sequence = args.submit_sequence; + self.context_id = args.registers.context_id.into(); + self.work_state_va = args.work_state_va; + self.checkpoints = args.registers.scratch.checkpoints().map(|n| n + if args.registers.matrix_ops { 2 } else { 0 }); + super::set_register_array(&mut self.registers, main_va, &main[..count])?; + super::set_register_array(&mut self.alternate, alternate_va, &alternate)?; + self.preempt_va = args.registers.preempt_va; + self.cdm_end_va = args.cdm_end_va; + self.control = 0x154024201; + self.predecessor_id = (args.registers.object_ids >> 32) as u32; + self.unk_f28 = u32::MAX; + self.stamp_va = args.stamp_va; + self.stamp_value = args.stamp_value; + self.qid = args.qid; + self.kick_timestamp = args.kick_timestamp; + self.current_id = args.registers.object_ids as u32 as u64; + self.completion_ordinal = args.completion_ordinal; + self.status_va = args.status_va; + self.timestamp_va = args.timestamp_va; + self.shared_control_va = args.shared_control_va; + self.shared_control_enabled = u8::from(args.shared_control_va != 0); + self.event_metadata = args.event_metadata; + self.usage_fw_va = args.usage_fw_va; + self.completion_identity = args.completion_identity; + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs b/drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs new file mode 100644 index 0000000000000..ed397b27917cb --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs @@ -0,0 +1,2867 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! M5 firmware initialization encodings and allocation layout. +//! Opaque initialized words have no inferred meaning. Runtime ownership is +//! provided by the common G17 KernelObject and GPUVM paths. + +#![allow(dead_code)] + +use core::ops::Range; +use core::result::Result; + +pub(crate) const ROOT_SIZE_PRIMARY: usize = 0x0b8; + +pub(crate) const ROOT_SIZE_SECONDARY: usize = 0x0c8; + +pub(crate) const MAIN_CONFIG_SIZE: usize = 0x600; + +pub(crate) const G17P_QOS_RESOURCE_SIZE: usize = 0xe40; + +pub(crate) const REGION_C_SIZE: usize = 0x1000; + +pub(crate) const STATUS_BLOCK_SIZE: usize = 0x80; + +pub(crate) const HW_DATA_BUNDLE_MIN_ALLOC: usize = 0x10000; + +pub(crate) const INITDATA_VERSION_WORD: u64 = 0x0c8a_a322_0396_04c0; + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum RootProfile { + J700G17P, + + J813G17G25G76, + + J873G18G26A428, + + J714G17S26A428, +} + +impl RootProfile { + pub(crate) const fn version_word(self) -> u64 { + match self { + Self::J700G17P => INITDATA_VERSION_WORD, + Self::J813G17G25G76 => 0x0c8a_a322_039e_04c0, + Self::J873G18G26A428 => COMMON26_INITDATA_VERSION_WORD, + Self::J714G17S26A428 => M5_INITDATA_VERSION_WORD, + } + } + + pub(crate) const fn root_size(self, role: InstanceRole) -> usize { + match self { + Self::J873G18G26A428 => COMMON26_ROOT_SIZE, + Self::J714G17S26A428 => M5_ROOT_SIZE, + Self::J700G17P | Self::J813G17G25G76 => root_size(role), + } + } + + pub(crate) const fn control_opening_opcode(self, role: InstanceRole) -> u32 { + match (self, role) { + (Self::J873G18G26A428, InstanceRole::Primary) => { + COMMON26_CONTROL_OPENING_OPCODE_PRIMARY + } + (Self::J873G18G26A428, InstanceRole::Secondary) => { + COMMON26_CONTROL_OPENING_OPCODE_SECONDARY + } + (Self::J714G17S26A428, InstanceRole::Primary) => M5_CONTROL_OPENING_OPCODE_PRIMARY, + (Self::J714G17S26A428, InstanceRole::Secondary) => { + M5_CONTROL_OPENING_OPCODE_SECONDARY + } + (_, InstanceRole::Primary) => CONTROL_OPENING_OPCODE_PRIMARY, + (_, InstanceRole::Secondary) => CONTROL_OPENING_OPCODE_SECONDARY, + } + } + + pub(crate) const fn control_register_opcode(self) -> Option { + match self { + Self::J873G18G26A428 | Self::J714G17S26A428 => None, + Self::J700G17P | Self::J813G17G25G76 => Some(CONTROL_OPENING_PRIMARY_REGISTER_OPCODE), + } + } + + pub(crate) const fn bundle_view_offsets(self) -> [usize; 5] { + match self { + Self::J873G18G26A428 => COMMON26_BUNDLE_VIEW_OFFSETS, + Self::J714G17S26A428 => M5_BUNDLE_VIEW_OFFSETS, + Self::J700G17P | Self::J813G17G25G76 => BUNDLE_VIEW_OFFSETS, + } + } + + pub(crate) const fn bundle_view_extents(self) -> [usize; 5] { + match self { + Self::J873G18G26A428 => COMMON26_BUNDLE_VIEW_EXTENTS, + Self::J714G17S26A428 => M5_BUNDLE_VIEW_EXTENTS, + Self::J700G17P | Self::J813G17G25G76 => BUNDLE_VIEW_EXTENTS, + } + } + + pub(crate) const fn repeated_region_bundle_offset(self) -> usize { + match self { + Self::J873G18G26A428 => COMMON26_SHARED_LAYOUT.block_88, + Self::J714G17S26A428 => M5_SHARED_LAYOUT.block_88, + Self::J700G17P | Self::J813G17G25G76 => MAIN_REPEATED_REGION_BUNDLE_OFFSET, + } + } + + pub(crate) const fn work_ring_bundle_offsets(self) -> [usize; WORK_CHANNEL_COUNT] { + match self { + Self::J873G18G26A428 => COMMON26_WORK_RING_BUNDLE_OFFSETS, + Self::J714G17S26A428 => M5_WORK_RING_BUNDLE_OFFSETS, + Self::J700G17P | Self::J813G17G25G76 => WORK_RING_BUNDLE_OFFSETS, + } + } +} + +const INITDATA_ADDR_MASK: u64 = 0x0000_0fff_ffff_ffff; + +const INITDATA_ADDR_PREFIX: u64 = 0xffff_f000_0000_0000; + +const ROOT_VERSION: usize = 0x00; + +const ROOT_REGION_A: usize = 0x08; + +const ROOT_MAIN_CONFIG: usize = 0x18; + +const ROOT_REGION_C: usize = 0x20; + +const ROOT_INSTANCE_KIND: usize = 0x28; + +const ROOT_FIXED_WORD: usize = 0x2c; + +const ROOT_PAGE_SIZE: usize = 0x30; + +const ROOT_PAGE_BITS: usize = 0x32; + +const ROOT_LEVEL_COUNT: usize = 0x33; + +const ROOT_LEVELS: usize = 0x34; + +const ROOT_STATUS_A: usize = 0xa8; + +const ROOT_STATUS_B: usize = 0xb0; + +const ROOT_SECONDARY_EXTRA_0: usize = 0xb8; + +const ROOT_SECONDARY_EXTRA_1: usize = 0xc0; + +pub(crate) const UAT_PAGE_SIZE: u16 = 0x4000; + +pub(crate) const UAT_PAGE_BITS: u8 = 14; + +pub(crate) const UAT_LEVEL_COUNT: usize = 3; + +pub(crate) const UAT_LEVELS: [(u8, u16); UAT_LEVEL_COUNT] = [(36, 64), (25, 2048), (14, 2048)]; + +pub(crate) const UAT_LEVEL_DESC_SIZE: usize = 0x20; + +pub(crate) const UAT_LEVEL_PHYS_MASK: u64 = 0x0000_03ff_ffff_c000; + +const MAIN_HW_DATA: usize = 0x00; + +const MAIN_REPEATED_0: usize = 0x08; + +const MAIN_REPEATED_1: usize = 0x10; + +pub(crate) const MAIN_REPEATED_REGION_BUNDLE_OFFSET: usize = 0xc500; + +const MAIN_CHANNEL_TABLE: usize = 0x20; + +const MAIN_BUNDLE_VIEWS: usize = 0x254; + +const MAIN_REGION_VIEWS: usize = 0x2d0; + +const MAIN_SECONDARY_KIND: usize = 0x300; + +const MAIN_SECONDARY_KIND_VALUE: u32 = 4; + +const MAIN_PRIMARY_MASK: usize = 0x3e0; + +const MAIN_PRIMARY_MASK_VALUE: u32 = 0xff; + +const MAIN_PRIMARY_SENTINEL_A: usize = 0x32c; + +const MAIN_PRIMARY_SENTINEL_B: usize = 0x344; + +const MAIN_SECONDARY_EXTRA_PTR: usize = 0x471; + +pub(crate) const SECONDARY_EXTRA_VIEW_INDEX: usize = 2; + +pub(crate) const SECONDARY_EXTRA_VIEW_OFFSET: usize = 0xe40; + +const MAIN_CONTROL_RING: usize = 0x4c0; + +pub(crate) const CONTROL_OPENING_OPCODE_PRIMARY: u32 = 0x16; + +pub(crate) const CONTROL_OPENING_OPCODE_SECONDARY: u32 = 0x2a; + +pub(crate) const CONTROL_RECORD_SIZE: usize = 0x40; + +pub(crate) const CONTROL_OPENING_PRIMARY_PRODUCER: u32 = 1; + +pub(crate) const CONTROL_OPENING_SECONDARY_PRODUCER: u32 = 1; + +pub(crate) const CONTROL_OPENING_PRIMARY_REGISTER_OPCODE: u32 = 0x20; + +pub(crate) const BUNDLE_VIEW_OFFSETS: [usize; 5] = [0x2740, 0x3380, 0x4400, 0xbc80, 0xbd00]; + +pub(crate) const BUNDLE_VIEW_EXTENTS: [usize; 5] = [0x18c0, 0x0c80, 0x3c00, 0x3380, 0x3300]; + +pub(crate) const REGION_VIEW_LOW_ADDRS: [u64; 2] = [0x70_0183_8000, 0x70_0184_0000]; + +pub(crate) const CHANNEL_TABLE_ENTRIES: usize = 17; + +pub(crate) const CHANNEL_ENTRY_SIZE: usize = 0x20; + +pub(crate) const CHANNEL_ENTRY_STATE_COUNT: usize = 3; + +pub(crate) const CHANNEL_STATE_SPACING: usize = 0x10; + +pub(crate) const CHANNEL_STATE_PRODUCER: usize = 2; + +pub(crate) const WORK_CHANNEL_COUNT: usize = 12; + +pub(crate) const CONTROL_CHANNEL_INDEX: usize = 12; + +pub(crate) const REPORT_CHANNEL_INDICES: [usize; 2] = [13, 14]; + +pub(crate) const PARTIAL_CHANNEL_INDEX: usize = 15; + +pub(crate) const WORK_STATE_GRID_OFFSETS: [usize; 13] = [ + 0x0c0, 0x1c0, 0x2c0, + 0x080, 0x180, 0x280, + 0x040, 0x140, 0x240, + 0x000, 0x100, 0x200, + 0x300, +]; + +pub(crate) const REPORT_STATE_STATUS_A_OFFSETS: [[usize; 3]; 2] = + [[0x00040, 0x002c0, 0x00080], [0x00240, 0xa6ac0, 0x00280]]; + +pub(crate) const REPORT_RING_STATUS_A_OFFSETS: [usize; 2] = [0x04ac0, 0xafac0]; + +pub(crate) const PARTIAL_STATE_STATUS_A_OFFSET: usize = 0x2d2c0; + +pub(crate) const WORK_RING_BUNDLE_OFFSETS: [usize; 12] = [ + 0x10dc0, 0x16dc0, 0x1cdc0, 0x0f5c0, 0x155c0, 0x1b5c0, 0x0ddc0, 0x13dc0, 0x19dc0, 0x0c5c0, + 0x125c0, 0x185c0, +]; + +pub(crate) const HW_REGISTER_MAP_OFFSET: usize = 0x640; + +pub(crate) const HW_REGISTER_MAP_ENTRY_SIZE: usize = 0x28; + +pub(crate) const HW_REGISTER_APERTURE: Range = 0xffff_fc21_8000_0000..0xffff_fc21_9000_0000; + +pub(crate) const G17P_KSM_COMPLETION_DESCRIPTOR_STRIDE: usize = 0x20; + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17PKsmCompletionAliases { + pub(crate) low: u64, + pub(crate) high: u64, +} + +pub(crate) const M5_25G76_COMPLETION_QUEUE_COUNT: usize = 4; + +pub(crate) const M5_25G76_COMPLETION_BACKING_SIZE: usize = 0x800 * 0x40; + +pub(crate) fn encode_m5_25g76_completion_table( + aliases: [G17PKsmCompletionAliases; M5_25G76_COMPLETION_QUEUE_COUNT], + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != M5_25G76_COMPLETION_QUEUE_COUNT * 0x20 { + return Err(InitdataError::BufferSize); + } + let extent = M5_25G76_COMPLETION_BACKING_SIZE as u64; + + for (i, alias) in aliases.iter().enumerate() { + for address in [alias.low, alias.high] { + required_address(address)?; + let end = address.checked_add(extent).ok_or(InitdataError::AddressOutOfRange)?; + required_address(end - 1)?; + } + for prior in &aliases[..i] { + if (alias.low < prior.low + extent && prior.low < alias.low + extent) + || (alias.high < prior.high + extent && prior.high < alias.high + extent) + { + return Err(InitdataError::PairAliasedObject); + } + } + } + out.fill(0); + for (ordinal, alias) in aliases.iter().enumerate() { + let offset = ordinal * 0x20; + put_u64(out, offset, alias.low); + put_u64(out, offset + 8, alias.high); + put_u32(out, offset + 0x10, 0x800); + put_u32(out, offset + 0x14, 0x40); + put_u32(out, offset + 0x18, ordinal as u32); + } + Ok(()) +} + +pub(crate) const STATUS_BLOCK_ONE_OFFSET: usize = 0x04; + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +#[repr(u32)] +pub(crate) enum InstanceRole { + + Primary = 0, + + Secondary = 1, +} + +pub(crate) const fn root_size(role: InstanceRole) -> usize { + match role { + InstanceRole::Primary => ROOT_SIZE_PRIMARY, + InstanceRole::Secondary => ROOT_SIZE_SECONDARY, + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct SharedAddresses { + + pub(crate) region_a: u64, + + pub(crate) region_c: u64, + + pub(crate) hw_data_bundle: u64, + + pub(crate) hw_data_bundle_alloc: usize, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct InstanceAddresses { + + pub(crate) root: u64, + pub(crate) main_config: u64, + pub(crate) status_a: u64, + + pub(crate) status_b: u64, + + pub(crate) secondary_extras: [u64; 2], +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct ChannelEntry { + pub(crate) states: [u64; 3], + pub(crate) ring: u64, +} + +impl ChannelEntry { + pub(crate) const EMPTY: ChannelEntry = ChannelEntry { + states: [0; 3], + ring: 0, + }; +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct PrimaryRegionViews { + pub(crate) sentinel_high: u64, + pub(crate) alias_high: [u64; 2], +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct RegisterWindow { + pub(crate) slot: usize, + pub(crate) phys: u64, + pub(crate) device_va: u64, + + pub(crate) size: u32, + + pub(crate) unk_18: u64, + pub(crate) flag: u32, +} + +pub(crate) const M5_25G76_HW_DATA_SIZE: usize = 0x2708; + +pub(crate) const M5_25G76_AON_PTD_SIZE: usize = 0xe4; + +pub(crate) fn encode_m5_25g76_aon_ptd(out: &mut [u8]) -> Result<(), InitdataError> { + if out.len() != M5_25G76_AON_PTD_SIZE { + return Err(InitdataError::BufferSize); + } + out.fill(0); + put_u32(out, 0, 0x6730_5452); + put_u32(out, 4, 0x001d_0001); + put_u32(out, 8, 0x0000_1401); + Ok(()) +} + +const M5_25G76_STATUS_CONFIG_WORDS: &[(usize, u32)] = &[ + (0x010, 0x0000001f), (0x014, 0x3e0a3d71), (0x018, 0x401ae148), + (0x01c, 0x00000028), (0x020, 0x00000001), (0x0c8, 0x00002af8), + (0x0cc, 0x00001f40), (0x0d8, 0x41980000), (0x0dc, 0x3d0f5c28), + (0x0f0, 0x000026ac), (0x0f4, 0x000000c8), (0x108, 0x42c80000), + (0x10c, 0x43c80000), (0x110, 0x000000c8), (0x128, 0x00000001), + (0x12c, 0x00005eb0), (0x130, 0x3e800000), (0x134, 0x3f99999a), + (0x158, 0x0000000e), (0x1c0, 0x00000001), (0x1c4, 0x00000001), + (0x1c8, 0x00000002), (0x1cc, 0x00000001), (0x1d0, 0x00000001), + (0x1d4, 0x00000001), (0x1d8, 0x00000001), (0x588, 0x00000001), + (0x58c, 0x0000000a), (0x590, 0x00000dac), (0x594, 0x000003e8), + (0x598, 0x00000bb8), (0x59c, 0x000002bc), (0x5cc, 0x00000001), + (0x5d0, 0x00000004), (0x5dc, 0x00001f40), (0x5e0, 0x000000c8), + (0x5e4, 0x00000fa0), (0x5e8, 0x0000012c), (0x5ec, 0x000007d0), + (0x5f0, 0x0000012c), (0x5f4, 0x000003e8), (0x5f8, 0x0000012c), + (0x600, 0x00000001), (0x604, 0x41700000), (0x608, 0x40a00000), + (0x60c, 0x00000020), (0x610, 0x000004b0), (0x620, 0x0000044c), +]; + +#[derive(Clone, Copy)] +pub(crate) struct M5PolicyBindings { + pub state: u64, + pub aon_ptd: u64, + pub boot_timestamp: u64, +} + +const M5_25G76_POLICY_WORDS: &[(usize, u32)] = &[ + (0x3d98, 0xffffffff), (0x3db0, 0xffffffff), (0x5244, 0x00030d40), + (0x5248, 0x00030d40), (0x5250, 0x00000004), (0x5254, 0x3f800000), + (0x526c, 0x00000001), (0x5270, 0x00000001), (0x5280, 0x00000064), + (0x5284, 0x00000001), (0x5288, 0x000004b0), (0x5290, 0x00000064), + (0x52a0, 0x00000004), (0x52a4, 0x3f800000), (0x52ac, 0x00000001), + (0x52b0, 0x00000001), (0x52c0, 0x00000064), (0x52c4, 0x00000001), + (0x52c8, 0x000004b0), (0x52d0, 0x00000064), (0x5300, 0x3f828f5c), + (0x5304, 0x3f828f5c), (0x5308, 0x3f828f5c), (0x530c, 0x3f828f5c), + (0x5310, 0x3f828f5c), (0x5314, 0x3f828f5c), (0x5318, 0x3f828f5c), + (0x531c, 0x3f828f5c), (0x5320, 0x3f828f5c), (0x5324, 0x3f828f5c), + (0x5328, 0x3f828f5c), (0x532c, 0x3f828f5c), (0x5330, 0x3f828f5c), + (0x5bf8, 0x00000271), (0x5c04, 0x3f7f2e9f), (0x5c0c, 0x3b516154), + (0x5c14, 0x3ca59586), (0x5c20, 0x46bd6000), (0x5c24, 0x40a90fe4), + (0x5c2c, 0xbd4ac5b4), (0x5c30, 0x00000028), (0x5c34, 0x000004b0), + (0x5c38, 0x000004b0), (0x5c44, 0x00005eb0), (0x5c78, 0x000004b0), + (0x5cb0, 0x00000008), (0x5cbc, 0x3f000000), (0x5cc4, 0x3f000000), + (0x5ccc, 0x3f99999a), (0x5cd8, 0x47800000), (0x5cdc, 0x3e800000), + (0x5ce8, 0x00000028), (0x5cec, 0x000004b0), (0x5cfc, 0x00005eb0), + (0x5d08, 0x00000010), (0x5d10, 0x0005dc00), (0x5d68, 0x0000004a), + (0x5d70, 0x0000005a), (0x5d74, 0x00000032), (0x5d78, 0x00000006), + (0x5d80, 0x00000006), (0x5d84, 0x00000001), (0x5d8c, 0x3f4ccccd), + (0x5d90, 0x3f7eb852), (0x5d94, 0x3e4ccccd), (0x5d98, 0x3ba3d70a), + (0x5d9c, 0x3f144b3c), (0x5da0, 0x3f144b3c), (0x5da8, 0x42be0000), + (0x5dac, 0x4029eb85), (0x5db0, 0x4029eb85), (0x5db4, 0x4139435e), + (0x5db8, 0x00000064), (0x5dbc, 0x000004b0), (0x5dc0, 0x00000064), + (0x5dcc, 0x0000004a), (0x5dd4, 0x00000001), (0x5e00, 0x00000064), + (0x5e20, 0x00000401), (0x5e24, 0x00006400), (0x5e2c, 0x001f4000), + (0x5e30, 0x0000c800), (0x5e34, 0x000fa000), (0x5e38, 0x00012c00), + (0x5e3c, 0x0007d000), (0x5e40, 0x00012c00), (0x5e44, 0x0003e800), + (0x5e48, 0x00012c00), (0x5e4c, 0x00000100), (0x5ea4, 0x00000001), + (0x5ed0, 0x00000001), (0x74d8, 0x47800000), (0x74dc, 0x42200000), + (0x74e0, 0x44960000), (0x74e4, 0x0000062a), (0x74f8, 0x00000028), + (0x7500, 0x42c80000), (0x7504, 0x000004b0), (0x750c, 0x3f4ccccd), + (0x7510, 0x3e4ccccd), (0x7550, 0xdcdcfbfa), (0x7558, 0x00001f40), + (0x7560, 0x00000004), (0x7574, 0x3f800000), (0x757c, 0x3d0f5c28), + (0x7588, 0x47800000), (0x758c, 0x41980000), (0x7598, 0x00000028), + (0x759c, 0x000004b0), (0x75a0, 0x000004b0), (0x75ac, 0x00002af8), + (0x75e0, 0x000004b0), (0x77f8, 0x00000002), (0x77fc, 0x42000006), + (0x7a10, 0xb8520000), (0x7a14, 0x00003f7e), (0x7a18, 0xd70a0000), + (0x7a1c, 0x00003ba3), (0x7a24, 0x000043c8), (0x7a30, 0x00004780), + (0x7a34, 0x000042c8), (0x7a3c, 0x0384bac8), (0x7a40, 0x03e80000), + (0x7a44, 0x03e80000), (0x7a4c, 0x80000000), (0x7a50, 0x26ac453b), + (0x7a84, 0x03e80000), (0x7aa0, 0x00c80000), (0x7b58, 0x000004b0), + (0x7b6c, 0x00000001), (0x7ba0, 0x00000101), (0x7ba4, 0x00000200), + (0x7bbc, 0x00023f00), (0x7bc0, 0x00000100), (0x7bc4, 0x00000100), + (0x7bdc, 0x00024000), (0x7be0, 0x00000100), (0x7be4, 0x00000100), + (0x7bfc, 0x00024100), (0x7c48, 0x00000001), (0x7c50, 0x00005eb0), + (0x7c54, 0x00005eb0), (0x7c58, 0x00005eb0), (0x7c68, 0x3f800000), + (0x7c6c, 0x00000004), (0x7c70, 0x00000008), (0x7c74, 0x0002ee00), + (0x7d44, 0x0000007c), (0x7d50, 0x3f77bdef), (0x7d58, 0x3d042108), + (0x7d60, 0x3e0a3d71), (0x7d6c, 0x47800000), (0x7d70, 0x401ae148), + (0x7d7c, 0x00000028), (0x7d80, 0x000004b0), (0x7d84, 0x000004b0), + (0x7d8c, 0x46bd6000), (0x7d90, 0x00005eb0), (0x7d9c, 0x000000f8), + (0x7da4, 0x005ad200), (0x7dc4, 0x000004b0), (0x7e04, 0x00000001), + (0x7ed4, 0x00000032), (0x7ed8, 0x00000001), (0x7ee0, 0x3f638e39), + (0x7ee4, 0x3f2aaaab), (0x7ee8, 0x3de38e39), (0x7eec, 0x3eaaaaab), + (0x7ef0, 0xbecccccd), (0x7ef4, 0xbf4ccccd), (0x7efc, 0x47800000), + (0x7f00, 0xc0a00000), (0x7f04, 0xc0a00000), (0x7f0c, 0x00000064), + (0x7f10, 0x000004b0), (0x7f14, 0x00000064), (0x7f1c, 0x45fa0000), + (0x7f20, 0x00000578), (0x7f28, 0x0000000e), (0x7f2c, 0x00000048), + (0x7f30, 0x00000018), (0x7f34, 0x001a5e00), (0x7f3c, 0x0008ca00), + (0x7f48, 0x47800000), (0x7f70, 0x00001f40), (0x7f74, 0x00000004), + (0x7f78, 0x000009c4), (0x7f7c, 0x00000258), (0x7f80, 0x00000002), + (0x7f84, 0x00000004), (0x7f88, 0x00000032), (0x7fb8, 0x47800000), + (0x7fc8, 0x00000028), (0x7fcc, 0x000004b0), (0x8098, 0x00000001), + (0x80ac, 0x0000000a), (0x80b8, 0x00000001), (0x8174, 0xac000000), + (0x8178, 0xe800000d), (0x817c, 0xb8000003), (0x8180, 0xb000000b), + (0x8184, 0xb0000004), (0x8188, 0xbc000004), (0x818c, 0x20000002), + (0x8198, 0x00000001), (0x81b4, 0x00000004), (0x81c0, 0x3f780000), + (0x81c8, 0x3d000000), (0x81d0, 0x40a00000), (0x81dc, 0x47800000), + (0x81e0, 0x41700000), (0x81ec, 0x0000044c), (0x81f0, 0x000004b0), + (0x81f4, 0x000004b0), (0x8200, 0x000004b0), (0x8234, 0x000004b0), + (0x827c, 0x00000064), (0x8290, 0xdcdcfbfa), (0x8298, 0x0000007d), + (0x829c, 0x00000001), (0x8ee0, 0xdcdcfbfb), (0x8ee8, 0x00000001), + (0xaeec, 0x41c80000), (0xaef0, 0x41c80000), (0xb6a8, 0x67305452), + (0xb6e8, 0x00000001), (0xb728, 0x5243464c), (0xb72c, 0x4758464c), + (0xb738, 0x47584c50), (0xb748, 0x00000001), (0xb7a8, 0x00000001), + (0xb7b4, 0x42b73333), (0xb7f4, 0x449b8000), (0xb834, 0x42bc0000), + (0xb8e0, 0x0000186a), +]; + +const M5_25G76_HW_PLATFORM_WORDS: &[(usize, u32)] = &[ + (0x0004, 0x0000006f), (0x0008, 0xffc00000), (0x0014, 0x00000010), + (0x001c, 0x00000010), (0x0020, 0xffff8000), (0x0024, 0x000002ff), + (0x0028, 0x81400000), (0x002c, 0xfffffc21), (0x00e0, 0x00002008), + (0x00e8, 0x20080000), (0x00f4, 0x00002008), (0x00f8, 0x000024cb), + (0x00fc, 0xc98a2cfa), (0x0100, 0xf4f624cb), (0x0104, 0x1877e917), + (0x0108, 0x38d924cb), (0x010c, 0xbdab0000), (0x02d8, 0xfff82000), + (0x02dc, 0xd32d2cdb), (0x02e0, 0xf5002000), (0x02e4, 0x21dae926), + (0x02e8, 0x38b62000), (0x02ec, 0xc7420008), (0x03e0, 0x00007fe0), + (0x03e8, 0x7fe00000), (0x03f4, 0x00007fe0), (0x03f8, 0x00008000), + (0x0400, 0x80000000), (0x040c, 0x00008000), (0x05d8, 0x4b232645), + (0x05dc, 0x00000e98), (0x05e0, 0xd5a2ea5f), (0x05e4, 0x40003fff), + (0x05e8, 0xca5e4000), (0x05ec, 0x4000f5a2), (0x0e90, 0x00008142), + (0x0e94, 0x00000002), (0x0ea0, 0x00000001), (0x0ea4, 0x00000001), + (0x0eb8, 0x00000001), (0x0ec0, 0x00000001), (0x0ec8, 0x00000001), + (0x0ed0, 0x00005dc0), (0x0ed4, 0x00000001), (0x0ed8, 0x00000008), + (0x0ee0, 0x00000001), (0x0ee4, 0x00000001), (0x0ee8, 0x00000001), + (0x0f04, 0x0000001f), (0x0f24, 0x07800004), (0x0f28, 0x0000000c), + (0x0f34, 0x00000001), (0x0f38, 0x00000001), (0x0f4c, 0x00000031), + (0x0f6c, 0x00000001), (0x0f88, 0x0000000a), (0x0f8c, 0x0000000f), + (0x0fac, 0x00000001), (0x0fb8, 0x00000020), (0x0fbc, 0x00000005), + (0x0fc0, 0x0000000a), (0x0fc4, 0x0000000c), (0x2544, 0x00000004), + (0x2548, 0x00000006), (0x254c, 0x00000003), (0x2550, 0x00000007), + (0x2554, 0x00000007), (0x2568, 0x0000000a), (0x2570, 0x00000001), + (0x258c, 0x00180000), (0x2594, 0x00000001), (0x2598, 0x00000001), + (0x259c, 0x00000001), (0x25a4, 0x00000001), (0x25a8, 0x00000001), + (0x25ac, 0x00000001), (0x25b4, 0xffffffff), (0x25b8, 0xffffffff), + (0x25bc, 0xffffffff), (0x25c0, 0xffffffff), (0x25c4, 0xffffffff), + (0x25c8, 0xffffffff), (0x25cc, 0xffffffff), (0x25d0, 0xffffffff), + (0x25d4, 0xffffffff), (0x25d8, 0xffffffff), (0x25dc, 0xffffffff), + (0x25e0, 0xffffffff), (0x25f4, 0x00000001), (0x2600, 0x00000001), + (0x2608, 0x00000001), (0x2610, 0x00000100), (0x26b4, 0x00000001), + (0x26b8, 0x00000001), (0x26e0, 0x000003e8), (0x26e8, 0x00000001), +]; + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum InitdataError { + BufferSize, + AddressOutOfRange, + NullRequiredAddress, + RoleSpecificAddress, + BundleAllocationTooSmall, + ChannelTableShape, + RegisterWindow, + PairRootSeparation, + PairAliasedObject, + SharedRegionSeparation, + InterfaceMismatch, + HostMappedAllocationsDisabled, +} + +fn address_representable(address: u64) -> bool { + let high = address & !INITDATA_ADDR_MASK; + high == 0 || high == INITDATA_ADDR_PREFIX +} + +fn required_address(address: u64) -> Result<(), InitdataError> { + if address == 0 { + Err(InitdataError::NullRequiredAddress) + } else if !address_representable(address) { + Err(InitdataError::AddressOutOfRange) + } else { + Ok(()) + } +} + +fn put_u16(bytes: &mut [u8], offset: usize, value: u16) { + bytes[offset..offset + 2].copy_from_slice(&value.to_le_bytes()); +} + +fn put_u32(bytes: &mut [u8], offset: usize, value: u32) { + bytes[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); +} + +fn put_u64(bytes: &mut [u8], offset: usize, value: u64) { + bytes[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); +} + +fn write_runs(bytes: &mut [u8], runs: &[(usize, &[u8])]) { + for (offset, data) in runs { + bytes[*offset..*offset + data.len()].copy_from_slice(data); + } +} + +fn validate_shared(shared: &SharedAddresses) -> Result<(), InitdataError> { + required_address(shared.region_a)?; + required_address(shared.region_c)?; + required_address(shared.hw_data_bundle)?; + if shared.hw_data_bundle_alloc < HW_DATA_BUNDLE_MIN_ALLOC { + return Err(InitdataError::BundleAllocationTooSmall); + } + Ok(()) +} + +fn validate_instance(role: InstanceRole, inst: &InstanceAddresses) -> Result<(), InitdataError> { + required_address(inst.root)?; + required_address(inst.main_config)?; + required_address(inst.status_a)?; + match role { + InstanceRole::Primary => { + + if inst.secondary_extras != [0, 0] { + return Err(InitdataError::RoleSpecificAddress); + } + required_address(inst.status_b)?; + } + InstanceRole::Secondary => { + + if inst.status_b != 0 { + return Err(InitdataError::RoleSpecificAddress); + } + for extra in inst.secondary_extras { + required_address(extra)?; + } + } + } + Ok(()) +} + +fn encode_uat_level_descriptor(bytes: &mut [u8], offset: usize, shift: u8, entries: u16) { + bytes[offset] = 8; + bytes[offset + 1] = UAT_PAGE_BITS; + bytes[offset + 2] = UAT_PAGE_BITS; + bytes[offset + 3] = shift; + put_u16(bytes, offset + 0x04, entries); + put_u16(bytes, offset + 0x06, UAT_PAGE_SIZE); + put_u64(bytes, offset + 0x08, 1); + put_u64(bytes, offset + 0x10, UAT_LEVEL_PHYS_MASK); + put_u64(bytes, offset + 0x18, (entries as u64 - 1) << shift); +} + +pub(crate) fn encode_root_for_profile( + profile: RootProfile, + role: InstanceRole, + shared: &SharedAddresses, + inst: &InstanceAddresses, + out: &mut [u8], +) -> Result<(), InitdataError> { + + if out.len() != profile.root_size(role) { + return Err(InitdataError::BufferSize); + } + validate_shared(shared)?; + validate_instance(role, inst)?; + + out.fill(0); + put_u64(out, ROOT_VERSION, profile.version_word()); + put_u64(out, ROOT_REGION_A, shared.region_a); + + put_u64(out, ROOT_MAIN_CONFIG, inst.main_config); + put_u64(out, ROOT_REGION_C, shared.region_c); + put_u32(out, ROOT_INSTANCE_KIND, role as u32); + put_u32(out, ROOT_FIXED_WORD, 1); + put_u16(out, ROOT_PAGE_SIZE, UAT_PAGE_SIZE); + out[ROOT_PAGE_BITS] = UAT_PAGE_BITS; + out[ROOT_LEVEL_COUNT] = UAT_LEVEL_COUNT as u8; + for (index, (shift, entries)) in UAT_LEVELS.iter().enumerate() { + encode_uat_level_descriptor( + out, + ROOT_LEVELS + index * UAT_LEVEL_DESC_SIZE, + *shift, + *entries, + ); + } + put_u64(out, ROOT_STATUS_A, inst.status_a); + put_u64(out, ROOT_STATUS_B, inst.status_b); + if role == InstanceRole::Secondary { + put_u64(out, ROOT_SECONDARY_EXTRA_0, inst.secondary_extras[0]); + put_u64(out, ROOT_SECONDARY_EXTRA_1, inst.secondary_extras[1]); + } + Ok(()) +} + +pub(crate) fn derive_channel_table( + role: InstanceRole, + state_grid: u64, + status_a: u64, + main_config: u64, + bundle: u64, +) -> [ChannelEntry; CHANNEL_TABLE_ENTRIES] { + let mut table = [ChannelEntry::EMPTY; CHANNEL_TABLE_ENTRIES]; + let grid_states = |index: usize| { + let base = state_grid + WORK_STATE_GRID_OFFSETS[index] as u64; + [ + base, + base + CHANNEL_STATE_SPACING as u64, + base + 2 * CHANNEL_STATE_SPACING as u64, + ] + }; + if role == InstanceRole::Primary { + for index in 0..WORK_CHANNEL_COUNT { + table[index] = ChannelEntry { + states: grid_states(index), + ring: bundle + WORK_RING_BUNDLE_OFFSETS[index] as u64, + }; + } + } + table[CONTROL_CHANNEL_INDEX] = ChannelEntry { + states: grid_states(CONTROL_CHANNEL_INDEX), + ring: main_config + MAIN_CONTROL_RING as u64, + }; + for (slot, index) in REPORT_CHANNEL_INDICES.iter().enumerate() { + let mut states = [0u64; 3]; + for (word, offset) in states.iter_mut().zip(REPORT_STATE_STATUS_A_OFFSETS[slot]) { + *word = status_a + offset as u64; + } + table[*index] = ChannelEntry { + states, + ring: status_a + REPORT_RING_STATUS_A_OFFSETS[slot] as u64, + }; + } + table[PARTIAL_CHANNEL_INDEX] = ChannelEntry { + states: [status_a + PARTIAL_STATE_STATUS_A_OFFSET as u64, 0, 0], + ring: 0, + }; + table +} + +pub(crate) fn derive_channel_table_for_profile( + profile: RootProfile, + role: InstanceRole, + state_grid: u64, + status_a: u64, + main_config: u64, + bundle: u64, +) -> [ChannelEntry; CHANNEL_TABLE_ENTRIES] { + let mut table = derive_channel_table(role, state_grid, status_a, main_config, bundle); + if role == InstanceRole::Primary { + let rings = profile.work_ring_bundle_offsets(); + for (index, entry) in table.iter_mut().take(WORK_CHANNEL_COUNT).enumerate() { + entry.ring = bundle + rings[index] as u64; + } + } + table +} + +fn validate_populated_entry(entry: &ChannelEntry) -> Result<(), InitdataError> { + + for state in entry.states { + if state == 0 { + return Err(InitdataError::ChannelTableShape); + } + if !address_representable(state) { + return Err(InitdataError::AddressOutOfRange); + } + } + required_address(entry.ring) +} + +fn validate_channel_table( + role: InstanceRole, + main_config: u64, + channels: &[ChannelEntry; CHANNEL_TABLE_ENTRIES], +) -> Result<(), InitdataError> { + for (index, entry) in channels.iter().enumerate() { + match index { + 0..=11 => match role { + InstanceRole::Primary => validate_populated_entry(entry)?, + + InstanceRole::Secondary => { + if *entry != ChannelEntry::EMPTY { + return Err(InitdataError::ChannelTableShape); + } + } + }, + 12 => { + validate_populated_entry(entry)?; + + if entry.ring != main_config + MAIN_CONTROL_RING as u64 { + return Err(InitdataError::ChannelTableShape); + } + } + 13 | 14 => validate_populated_entry(entry)?, + 15 => { + + required_address(entry.states[0])?; + if entry.states[1] != 0 || entry.states[2] != 0 || entry.ring != 0 { + return Err(InitdataError::ChannelTableShape); + } + } + _ => { + if *entry != ChannelEntry::EMPTY { + return Err(InitdataError::ChannelTableShape); + } + } + } + } + Ok(()) +} + +pub(crate) fn encode_main_config_for_profile( + profile: RootProfile, + role: InstanceRole, + shared: &SharedAddresses, + inst: &InstanceAddresses, + channels: &[ChannelEntry; CHANNEL_TABLE_ENTRIES], + region_views: Option<&PrimaryRegionViews>, + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != MAIN_CONFIG_SIZE { + return Err(InitdataError::BufferSize); + } + validate_shared(shared)?; + validate_instance(role, inst)?; + validate_channel_table(role, inst.main_config, channels)?; + match (role, region_views) { + (InstanceRole::Primary, Some(views)) => { + required_address(views.sentinel_high)?; + for high in views.alias_high { + required_address(high)?; + } + } + (InstanceRole::Secondary, None) => {} + _ => return Err(InitdataError::RoleSpecificAddress), + } + + let view_offsets = profile.bundle_view_offsets(); + for (offset, extent) in view_offsets.iter().zip(profile.bundle_view_extents()) { + if offset + extent > shared.hw_data_bundle_alloc { + return Err(InitdataError::BundleAllocationTooSmall); + } + } + + out.fill(0); + + let bundle = shared.hw_data_bundle; + put_u64(out, MAIN_HW_DATA, bundle); + let repeated = bundle + profile.repeated_region_bundle_offset() as u64; + put_u64(out, MAIN_REPEATED_0, repeated); + put_u64(out, MAIN_REPEATED_1, repeated); + + for (index, entry) in channels.iter().enumerate() { + let base = MAIN_CHANNEL_TABLE + index * CHANNEL_ENTRY_SIZE; + for (word, state) in entry.states.iter().enumerate() { + put_u64(out, base + word * 8, *state); + } + put_u64(out, base + CHANNEL_ENTRY_STATE_COUNT * 8, entry.ring); + } + + match (role, region_views) { + (InstanceRole::Primary, Some(views)) => { + + for (index, offset) in view_offsets.iter().enumerate() { + put_u64(out, MAIN_BUNDLE_VIEWS + index * 8, bundle + *offset as u64); + } + + put_u64(out, MAIN_REGION_VIEWS, views.sentinel_high); + put_u64(out, MAIN_REGION_VIEWS + 0x08, REGION_VIEW_LOW_ADDRS[0]); + put_u64(out, MAIN_REGION_VIEWS + 0x10, views.alias_high[0]); + put_u64(out, MAIN_REGION_VIEWS + 0x18, REGION_VIEW_LOW_ADDRS[1]); + put_u64(out, MAIN_REGION_VIEWS + 0x20, views.alias_high[1]); + // J714's firmware takes both primary sentinels too; they are not + // exclusive to the older J700 ABI. + if matches!(profile, RootProfile::J700G17P | RootProfile::J714G17S26A428) { + put_u32(out, MAIN_PRIMARY_SENTINEL_A, 1); + put_u32(out, MAIN_PRIMARY_SENTINEL_B, 0xabcd_abcd); + } + + put_u32(out, MAIN_PRIMARY_MASK, MAIN_PRIMARY_MASK_VALUE); + put_u32(out, MAIN_CONTROL_RING, profile.control_opening_opcode(role)); + } + (InstanceRole::Secondary, _) => { + + put_u64(out, MAIN_REGION_VIEWS + 0x08, REGION_VIEW_LOW_ADDRS[0]); + put_u64(out, MAIN_REGION_VIEWS + 0x18, REGION_VIEW_LOW_ADDRS[1]); + put_u32(out, MAIN_SECONDARY_KIND, match profile { + RootProfile::J700G17P => MAIN_SECONDARY_KIND_VALUE, + RootProfile::J813G17G25G76 => 0xc, + + RootProfile::J873G18G26A428 | RootProfile::J714G17S26A428 => { + COMMON26_MAIN_SECONDARY_KIND + } + }); + if profile == RootProfile::J714G17S26A428 { + + put_u64(out, COMMON26_MAIN_SECONDARY_POLICY_PTR, + bundle + M5_SHARED_LAYOUT.policy as u64); + } else if profile == RootProfile::J873G18G26A428 { + + put_u64(out, COMMON26_MAIN_SECONDARY_POLICY_PTR, + bundle + COMMON26_SHARED_LAYOUT.policy as u64); + } else { + + let extra = bundle + + BUNDLE_VIEW_OFFSETS[SECONDARY_EXTRA_VIEW_INDEX] as u64 + + SECONDARY_EXTRA_VIEW_OFFSET as u64; + put_u64(out, MAIN_SECONDARY_EXTRA_PTR, extra); + } + put_u32(out, MAIN_CONTROL_RING, profile.control_opening_opcode(role)); + } + _ => unreachable!(), + } + Ok(()) +} + +pub(crate) fn encode_region_c_for_profile( + profile: RootProfile, + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != REGION_C_SIZE { + return Err(InitdataError::BufferSize); + } + if profile == RootProfile::J873G18G26A428 { + + return encode_common26_region_c(out); + } + if profile == RootProfile::J714G17S26A428 { + + return Err(InitdataError::BufferSize); + } + out.fill(0); + write_runs(out, REGION_C_RUNS); + if profile == RootProfile::J813G17G25G76 { + + for (offset, value) in [ + (0x084, 0x0000_5eb0), + (0x088, 0x0000_04b0), + (0x08c, 0x0000_04b0), + (0x098, 0x0000_04b0), + (0x7c4, 0x0000_04e2), + (0x7e0, 0x3f33_3333), + (0x9a8, 0x0000_0333), + (0xe38, 0), + (0xe3c, 0), + (0xe58, 0x0000_0300), + (0xe5c, 0x0000_0100), + ] { + put_u32(out, offset, value); + } + return Ok(()); + } + + if 1 == 0 { + put_u32(out, 0x9cc, 0); + } + + if 0 != 0 { + out[0xe21] = 1; + } + for (offset, value) in [ + (0x7c, 1), + (0x80, 1), + (0x9c, 0x3000_001d), + (0xa0, 0x3100_0002), + ] { + put_u32(out, offset, value); + } + Ok(()) +} + +pub(crate) fn encode_status_block(out: &mut [u8]) -> Result<(), InitdataError> { + if out.len() != STATUS_BLOCK_SIZE { + return Err(InitdataError::BufferSize); + } + out.fill(0); + put_u32(out, STATUS_BLOCK_ONE_OFFSET, 1); + Ok(()) +} + +pub(crate) fn encode_secondary_root_extra_1_for_profile( + profile: RootProfile, + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != STATUS_BLOCK_SIZE { + return Err(InitdataError::BufferSize); + } + out.fill(0); + write_runs(out, SECONDARY_ROOT_EXTRA_1_RUNS); + + if matches!(profile, RootProfile::J813G17G25G76 | RootProfile::J873G18G26A428) { + put_u32(out, 0x10, 2); + } + Ok(()) +} + +pub(crate) const REGION_C_RUNS: &[(usize, &[u8])] = &[ + (0x0024, &[0xb8, 0x0b, 0x00, 0x00]), + ( + 0x0030, + &[ + 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x78, 0x00, + 0x00, 0x00, + ], + ), + ( + 0x0054, + &[ + 0xff, 0xff, 0x28, 0x00, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x01, 0x00, + ], + ), + (0x0078, &[0x01, 0x00, 0x00, 0x00]), + ( + 0x0084, + &[ + 0x4e, 0x25, 0x00, 0x00, 0xe8, 0x03, 0x00, 0x00, 0xe8, 0x03, 0x00, 0x00, + ], + ), + (0x0098, &[0xe8, 0x03, 0x00, 0x00]), + ( + 0x07c4, + &[ + 0xd0, 0x07, 0x00, 0x00, 0x00, 0x00, 0x80, 0x3f, 0xcd, 0xcc, 0x4c, 0x3f, 0xcd, 0xcc, + 0x4c, 0x3e, 0x66, 0x66, 0x66, 0x3f, 0xcd, 0xcc, 0xcc, 0x3d, 0x00, 0x00, 0x80, 0x3e, + 0x9a, 0x99, 0x19, 0x3f, 0x66, 0x66, 0x66, 0x3f, 0x06, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x00, + ], + ), + ( + 0x0998, + &[ + 0x28, 0x00, 0x00, 0x00, 0x0a, 0x00, 0x00, 0x00, 0xfa, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x00, 0x00, + ], + ), + ( + 0x09b8, + &[ + 0x02, 0x00, 0x00, 0x00, 0x28, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x28, 0x00, 0x00, 0x00, 0x32, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + ], + ), + (0x0e1c, &[0x00, 0x01, 0x00, 0x00]), + (0x0e38, &[0x00, 0x03, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00]), + (0x0e50, &[0x00, 0x00, 0x00, 0x00]), +]; + +pub(crate) const SECONDARY_ROOT_EXTRA_1_RUNS: &[(usize, &[u8])] = &[ + (0x14, &[0x01]), + (0x2c, &[0x01]), + (0x30, &[0x01]), + (0x40, &[0x01]), + (0x4c, &[0x01]), + (0x50, &[0x6a, 0x18]), +]; + +pub(crate) const COMMON26_INITDATA_VERSION_WORD: u64 = 0x0c8b_4339_83a0_04c0; + +pub(crate) const COMMON26_ROOT_SIZE: usize = 0xc8; + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum Evidence { + FirmwareProven, + ProducerChecked, + AdtProven, + Inferred, + Guessed, +} + +pub(crate) const ABI_ALLOCATION_SIZES_25G76: [usize; 24] = [ + 0x2708, 0x0c18, 0x1048, 0x0e10, 0x6a30, 0x0068, 0x0800, 0x0088, 0x1800, 0x04c0, 0x4000, 0x0030, + 0xe440, 0x06a8, 0x0020, 0x01b0, 0x4800, 0x28800, 0x79800, 0x9000, 0x0060, 0x0008, 0x0e78, 0x00e4, +]; + +pub(crate) const ABI_ALLOCATION_SIZES_26A428: [usize; 24] = [ + 0x2848, 0x0c18, 0x1048, 0x0e10, 0x7800, 0x0068, 0x0800, 0x0088, 0x1800, 0x04c0, 0x4000, 0x0030, + 0xe730, 0x06d0, 0x0020, 0x01b0, 0x4800, 0x28800, 0x79800, 0x9000, 0x0060, 0x0008, 0x0e80, 0x00e4, +]; + +pub(crate) const ALLOC_HW_DATA: usize = 0; + +pub(crate) const ALLOC_BLOCK_C18: usize = 1; + +pub(crate) const ALLOC_BLOCK_1048: usize = 2; + +pub(crate) const ALLOC_BLOCK_E10: usize = 3; + +pub(crate) const ALLOC_POLICY: usize = 4; + +pub(crate) const ALLOC_BLOCK_68: usize = 5; + +pub(crate) const ALLOC_BLOCK_800: usize = 6; + +pub(crate) const ALLOC_BLOCK_88: usize = 7; + +pub(crate) const ALLOC_DM_RING: usize = 8; + +pub(crate) const ALLOC_MAIN_CONFIG: usize = 9; + +pub(crate) const ALLOC_CONTROL_RING: usize = 10; + +pub(crate) const ALLOC_RING_STATE: usize = 11; + +pub(crate) const ALLOC_STATUS_B: usize = 12; + +pub(crate) const ALLOC_STATUS_CONFIG: usize = 13; + +pub(crate) const ALLOC_STATUS_A: usize = 14; + +pub(crate) const ALLOC_LOG_STATE: usize = 15; + +pub(crate) const ALLOC_RING_4800: usize = 16; + +pub(crate) const ALLOC_BUFFER_28800: usize = 17; + +pub(crate) const ALLOC_BUFFER_79800: usize = 18; + +pub(crate) const ALLOC_RING_9000: usize = 19; + +pub(crate) const ALLOC_SECONDARY_EXTRA_1: usize = 20; + +pub(crate) const ALLOC_POLICY_STATE: usize = 21; + +pub(crate) const ALLOC_REGION_C: usize = 22; + +pub(crate) const ALLOC_AON_PTD: usize = 23; + +pub(crate) const ABI_BRN_TABLE_SIZE: usize = 0; + +pub(crate) const ABI_ALLOCATION_PAGE: usize = 0x4000; + +pub(crate) const SHARED_ALLOCATION_ENTRIES: usize = 25; + +pub(crate) const PRIVATE_ALLOCATION_ENTRIES: usize = 38; + +const ROLE_SET_ENTRIES: usize = 10; + +const fn round_up_usize(value: usize, alignment: usize) -> usize { + (value + alignment - 1) & !(alignment - 1) +} + +pub(crate) const fn next_packed_allocation(offset: usize, size: usize) -> usize { + round_up_usize(offset + size, 0x40) +} + +pub(crate) const fn packed_allocation_offsets(sizes: &[usize; N]) -> [usize; N] { + let mut out = [0usize; N]; + let mut index = 1; + while index < N { + out[index] = next_packed_allocation(out[index - 1], sizes[index - 1]); + index += 1; + } + out +} + +pub(crate) const fn shared_allocation_sizes(table: &[usize; 24]) -> [usize; SHARED_ALLOCATION_ENTRIES] { + let mut sizes = [0usize; SHARED_ALLOCATION_ENTRIES]; + sizes[0] = table[ALLOC_HW_DATA]; + sizes[1] = table[ALLOC_BLOCK_C18]; + sizes[2] = table[ALLOC_BLOCK_1048]; + sizes[3] = table[ALLOC_BLOCK_E10]; + sizes[4] = table[ALLOC_POLICY]; + sizes[5] = table[ALLOC_BLOCK_68]; + sizes[6] = table[ALLOC_BLOCK_800]; + sizes[7] = ABI_BRN_TABLE_SIZE; + sizes[8] = table[ALLOC_BLOCK_88]; + let mut ring = 0; + while ring < WORK_CHANNEL_COUNT { + sizes[9 + ring] = table[ALLOC_DM_RING]; + ring += 1; + } + sizes[21] = table[ALLOC_MAIN_CONFIG]; + sizes[22] = table[ALLOC_CONTROL_RING]; + sizes[23] = table[ALLOC_MAIN_CONFIG]; + sizes[24] = table[ALLOC_CONTROL_RING]; + sizes +} + +pub(crate) const fn private_allocation_sizes(table: &[usize; 24]) -> [usize; PRIVATE_ALLOCATION_ENTRIES] { + let mut sizes = [0usize; PRIVATE_ALLOCATION_ENTRIES]; + let mut state = 0; + while state <= WORK_CHANNEL_COUNT { + sizes[state] = table[ALLOC_RING_STATE]; + state += 1; + } + sizes[13] = table[ALLOC_STATUS_B]; + sizes[14] = table[ALLOC_STATUS_CONFIG]; + let role_set = [ + table[ALLOC_STATUS_A], table[ALLOC_RING_STATE], table[ALLOC_LOG_STATE], table[ALLOC_RING_STATE], + table[ALLOC_RING_STATE], table[ALLOC_RING_4800], table[ALLOC_BUFFER_28800], + table[ALLOC_BUFFER_79800], table[ALLOC_RING_9000], table[ALLOC_RING_4800], + ]; + let mut role = 0; + while role < 2 { + let mut entry = 0; + while entry < ROLE_SET_ENTRIES { + sizes[15 + role * ROLE_SET_ENTRIES + entry] = role_set[entry]; + entry += 1; + } + role += 1; + } + sizes[35] = table[ALLOC_RING_STATE]; + sizes[36] = table[ALLOC_SECONDARY_EXTRA_1]; + sizes[37] = table[ALLOC_POLICY_STATE]; + sizes +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct SharedClusterLayout { + pub(crate) hw_data: usize, + pub(crate) block_c18: usize, + pub(crate) block_1048: usize, + pub(crate) block_e10: usize, + pub(crate) policy: usize, + pub(crate) block_68: usize, + pub(crate) block_800: usize, + pub(crate) brn_table: usize, + pub(crate) block_88: usize, + + pub(crate) dm_rings: [usize; WORK_CHANNEL_COUNT], + pub(crate) main_config: [usize; 2], + pub(crate) control_ring: [usize; 2], + pub(crate) end: usize, + pub(crate) total: usize, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct RoleSetLayout { + + pub(crate) status_a: usize, + + pub(crate) event_state: usize, + + pub(crate) log_state: usize, + + pub(crate) trace_state: usize, + pub(crate) state_30: usize, + + pub(crate) event_ring: usize, + pub(crate) buffer_28800: usize, + + pub(crate) buffer_79800: usize, + + pub(crate) trace_ring: usize, + pub(crate) buffer_4800: usize, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct PrivateClusterLayout { + pub(crate) dm_ring_states: [usize; WORK_CHANNEL_COUNT], + + pub(crate) control_state: usize, + + pub(crate) status_b: usize, + + pub(crate) status_config: usize, + pub(crate) roles: [RoleSetLayout; 2], + + pub(crate) secondary_control_state: usize, + + pub(crate) secondary_extra_1: usize, + + pub(crate) policy_state: usize, + pub(crate) end: usize, + pub(crate) total: usize, +} + +pub(crate) const fn shared_cluster_layout(table: &[usize; 24]) -> SharedClusterLayout { + let sizes = shared_allocation_sizes(table); + let offsets = packed_allocation_offsets(&sizes); + let mut dm_rings = [0usize; WORK_CHANNEL_COUNT]; + let mut ring = 0; + while ring < WORK_CHANNEL_COUNT { + dm_rings[ring] = offsets[9 + ring]; + ring += 1; + } + let end = offsets[24] + sizes[24]; + SharedClusterLayout { + hw_data: offsets[0], + block_c18: offsets[1], + block_1048: offsets[2], + block_e10: offsets[3], + policy: offsets[4], + block_68: offsets[5], + block_800: offsets[6], + brn_table: offsets[7], + block_88: offsets[8], + dm_rings, + main_config: [offsets[21], offsets[23]], + control_ring: [offsets[22], offsets[24]], + end, + total: round_up_usize(end, ABI_ALLOCATION_PAGE), + } +} + +const fn role_set_layout(offsets: &[usize; PRIVATE_ALLOCATION_ENTRIES], first: usize) -> RoleSetLayout { + RoleSetLayout { + status_a: offsets[first], + event_state: offsets[first + 1], + log_state: offsets[first + 2], + trace_state: offsets[first + 3], + state_30: offsets[first + 4], + event_ring: offsets[first + 5], + buffer_28800: offsets[first + 6], + buffer_79800: offsets[first + 7], + trace_ring: offsets[first + 8], + buffer_4800: offsets[first + 9], + } +} + +pub(crate) const fn private_cluster_layout(table: &[usize; 24]) -> PrivateClusterLayout { + let sizes = private_allocation_sizes(table); + let offsets = packed_allocation_offsets(&sizes); + let mut dm_ring_states = [0usize; WORK_CHANNEL_COUNT]; + let mut state = 0; + while state < WORK_CHANNEL_COUNT { + dm_ring_states[state] = offsets[state]; + state += 1; + } + let end = offsets[37] + sizes[37]; + PrivateClusterLayout { + dm_ring_states, + control_state: offsets[12], + status_b: offsets[13], + status_config: offsets[14], + roles: [role_set_layout(&offsets, 15), role_set_layout(&offsets, 25)], + secondary_control_state: offsets[35], + secondary_extra_1: offsets[36], + policy_state: offsets[37], + end, + total: round_up_usize(end, ABI_ALLOCATION_PAGE), + } +} + +pub(crate) const WORK_CHANNEL_DM_INDEX: [usize; WORK_CHANNEL_COUNT] = + [3, 7, 11, 2, 6, 10, 1, 5, 9, 0, 4, 8]; + +pub(crate) const fn work_ring_offsets(layout: &SharedClusterLayout) -> [usize; WORK_CHANNEL_COUNT] { + let mut rings = [0usize; WORK_CHANNEL_COUNT]; + let mut channel = 0; + while channel < WORK_CHANNEL_COUNT { + rings[channel] = layout.dm_rings[WORK_CHANNEL_DM_INDEX[channel]]; + channel += 1; + } + rings +} + +pub(crate) const M5_25G76_SHARED_LAYOUT: SharedClusterLayout = + shared_cluster_layout(&ABI_ALLOCATION_SIZES_25G76); + +pub(crate) const COMMON26_SHARED_LAYOUT: SharedClusterLayout = + shared_cluster_layout(&ABI_ALLOCATION_SIZES_26A428); + +pub(crate) const COMMON26_HW_DATA_SIZE: usize = ABI_ALLOCATION_SIZES_26A428[ALLOC_HW_DATA]; + +pub(crate) const COMMON26_BUNDLE_VIEW_OFFSETS: [usize; 5] = [ + COMMON26_SHARED_LAYOUT.block_c18, + COMMON26_SHARED_LAYOUT.block_1048, + COMMON26_SHARED_LAYOUT.block_e10, + COMMON26_SHARED_LAYOUT.block_68, + COMMON26_SHARED_LAYOUT.block_800, +]; + +pub(crate) const COMMON26_BUNDLE_VIEW_EXTENTS: [usize; 5] = [ + ABI_ALLOCATION_SIZES_26A428[ALLOC_BLOCK_C18], + ABI_ALLOCATION_SIZES_26A428[ALLOC_BLOCK_1048], + ABI_ALLOCATION_SIZES_26A428[ALLOC_BLOCK_E10], + ABI_ALLOCATION_SIZES_26A428[ALLOC_BLOCK_68], + ABI_ALLOCATION_SIZES_26A428[ALLOC_BLOCK_800], +]; + +pub(crate) const COMMON26_WORK_RING_BUNDLE_OFFSETS: [usize; WORK_CHANNEL_COUNT] = + work_ring_offsets(&COMMON26_SHARED_LAYOUT); + +pub(crate) const COMMON26_MAIN_SECONDARY_POLICY_PTR: usize = 0x481; + +pub(crate) const COMMON26_MAIN_SECONDARY_KIND: u32 = 0xc; + +pub(crate) const COMMON26_FWCTL_RING_OFFSET: usize = 0x40; + +pub(crate) const COMMON26_STATUS_B_FWCTL_STATE: usize = 0x4ac0; + +pub(crate) const COMMON26_STATUS_B_FWCTL_RING: usize = 0x4ac8; + +pub(crate) const COMMON26_STATUS_B_CONFIG_HEADER: usize = 0xe714; + +pub(crate) const COMMON26_STATUS_B_LINK_WORD: usize = 0xe71c; + +pub(crate) const COMMON26_STATUS_B_LINK_PTR: usize = 0xe720; + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct LayoutBand { + pub(crate) start: usize, + pub(crate) end: usize, + pub(crate) delta: isize, + pub(crate) evidence: Evidence, +} + +const fn band(start: usize, end: usize, delta: isize, evidence: Evidence) -> LayoutBand { + LayoutBand { start, end, delta, evidence } +} + +pub(crate) const fn relocate_by_bands(bands: &[LayoutBand], offset: usize) -> Option { + let mut index = 0; + while index < bands.len() { + let band = bands[index]; + if band.start <= offset && offset < band.end { + return Some((offset as isize + band.delta) as usize); + } + index += 1; + } + None +} + +pub(crate) const COMMON26_HW_DATA_BANDS: &[LayoutBand] = &[ + + band(0x000, 0x640, 0, Evidence::Inferred), + + band(0xe88, 0xe90, 0x118, Evidence::ProducerChecked), + + band(0xe90, 0xebc, 0x120, Evidence::FirmwareProven), + + band(0xec0, 0xf5c, 0x11c, Evidence::FirmwareProven), + band(0xf5c, 0xf60, 0x124, Evidence::ProducerChecked), + + band(0xf68, 0xf98, 0x11c, Evidence::FirmwareProven), + band(0xf98, 0xfac, 0x120, Evidence::FirmwareProven), + + band(0xfb0, 0x25f8, 0x11c, Evidence::FirmwareProven), + band(0x25f8, 0x26b4, 0x120, Evidence::FirmwareProven), + band(0x26b4, 0x26f0, 0x128, Evidence::FirmwareProven), + band(0x26f0, M5_25G76_HW_DATA_SIZE, 0x12c, Evidence::FirmwareProven), +]; + +pub(crate) const COMMON26_STATUS_CONFIG_BANDS: &[LayoutBand] = &[ + band(0x000, 0x1dc, 0, Evidence::ProducerChecked), + band(0x4e0, 0x518, 0xc, Evidence::ProducerChecked), + band(0x518, 0x528, 0x14, Evidence::ProducerChecked), + band(0x530, 0x534, 0xc, Evidence::ProducerChecked), + band(0x554, 0x584, 0xc, Evidence::ProducerChecked), + band(0x584, 0x5a8, 0x1c, Evidence::ProducerChecked), + band(0x5cc, 0x628, 0x18, Evidence::ProducerChecked), +]; + +fn write_relocated_words( + words: &[(usize, u32)], + source_base: usize, + bands: &[LayoutBand], + out: &mut [u8], +) -> Result<(), InitdataError> { + for &(offset, value) in words { + for (index, byte) in value.to_le_bytes().into_iter().enumerate() { + if byte == 0 { + continue; + } + let source = offset + .checked_sub(source_base) + .ok_or(InitdataError::BufferSize)? + + index; + if let Some(target) = relocate_by_bands(bands, source) { + *out.get_mut(target).ok_or(InitdataError::BufferSize)? = byte; + } + } + } + Ok(()) +} + +pub(crate) const COMMON26_HW_REGISTER_MAP_SLOTS: usize = 60; + +pub(crate) const COMMON26_HW_REGISTER_MAP_END: usize = + HW_REGISTER_MAP_OFFSET + COMMON26_HW_REGISTER_MAP_SLOTS * HW_REGISTER_MAP_ENTRY_SIZE; + +pub(crate) const COMMON26_HW_CHIP_ID_OFFSET: usize = 0xfb0; + +pub(crate) const COMMON26_HW_PERF_STATE_COUNT: usize = 0x10e0; + +pub(crate) const COMMON26_HW_PERF_FREQUENCY: usize = 0x10e4; + +pub(crate) const COMMON26_HW_PERF_CORE_VOLTAGE: usize = 0x1124; + +pub(crate) const COMMON26_HW_PERF_SRAM_VOLTAGE: usize = 0x1524; + +pub(crate) const COMMON26_HW_PERF_ROWS: usize = 16; + +pub(crate) const COMMON26_HW_PERF_ROW_STRIDE: usize = 0x40; + +pub(crate) const COMMON26_HW_PERF_LANES: usize = 16; + +pub(crate) const COMMON26_HW_PERF_SCALE: usize = 0x1964; + +pub(crate) const COMMON26_HW_PERF_RELATIVE_A: usize = 0x19e4; + +pub(crate) const COMMON26_HW_PERF_RELATIVE_B: usize = 0x1a24; + +const M5_HW_AUX_RELATIVE_FREQUENCY: usize = 0x1aa4; +const M5_HW_AUX_PERF_STATE: usize = 0x1cac; +const M5_HW_AUX_PERF_STATE_SIZE: usize = 0x148; + +// Clock-domain 2: MHz, core mV, SRAM mV. The second voltage lane is unused. +const J714_AUX_PERF_STATES: [M5AuxPerfState; 14] = one_lane([ + (24, 125, 775), (338, 590, 775), (486, 620, 775), (636, 645, 775), + (796, 685, 775), (888, 700, 775), (988, 745, 775), (1084, 770, 775), + (1182, 795, 795), (1278, 835, 835), (1374, 860, 860), + (1470, 900, 900), (1578, 960, 960), (1620, 990, 990), +]); + +// J716c ADT afr-perf-states (identical to cs-perf-states). SRAM is +// max(775, core), the rule the J714 column follows. +const J716C_AUX_PERF_STATES: [M5AuxPerfState; 14] = one_lane([ + (24, 125, 775), (338, 590, 775), (486, 625, 775), (636, 640, 775), + (796, 690, 775), (888, 700, 775), (988, 740, 775), (1084, 765, 775), + (1182, 780, 780), (1278, 830, 830), (1374, 875, 875), + (1470, 910, 910), (1578, 975, 975), (1620, 1020, 1020), +]); + +/// An auxiliary-domain performance state: MHz, then core and SRAM mV of each of its two voltage +/// lanes. A one-lane board leaves the second lane 0. +pub(crate) type M5AuxPerfState = (u32, [u32; 2], [u32; 2]); + +// J775D ADT afr-perf-states / cs-perf-states: one lane per die, mV; SRAM floor 775 mV. +const J775D_AUX_PERF_STATES: [M5AuxPerfState; 14] = [ + (24, [125, 125], [775, 775]), (338, [600, 600], [775, 775]), + (486, [660, 655], [775, 775]), (636, [670, 665], [775, 775]), + (796, [710, 705], [775, 775]), (888, [720, 720], [775, 775]), + (988, [755, 765], [775, 775]), (1084, [800, 795], [800, 795]), + (1182, [810, 820], [810, 820]), (1278, [850, 860], [850, 860]), + (1374, [905, 915], [905, 915]), (1470, [945, 955], [945, 955]), + (1578, [1030, 1040], [1030, 1040]), (1620, [1075, 1085], [1075, 1085]), +]; + +/// One-lane auxiliary tables. +const fn one_lane(states: [(u32, u32, u32); 14]) -> [M5AuxPerfState; 14] { + let mut out = [(0, [0; 2], [0; 2]); 14]; + let mut state = 0; + while state < 14 { + let (frequency, core, sram) = states[state]; + out[state] = (frequency, [core, 0], [sram, 0]); + state += 1; + } + out +} + +// Evaluate the frequency-only curve at compile time; no kernel FPU use. +// Conversion uses Hz before the single-precision multiply and divide. The curve is based at +// `floor`, the lowest state the performance controllers run the GPU at: states at and below it +// read 0. +const fn relative_frequency(states: &[M5PerfState; 14], floor: usize) -> [u32; 16] { + let mut values = [0; 16]; + let base = states[floor].0 as u64 * 1_000_000; + let maximum = states[13].0 as u64 * 1_000_000; + let denominator = (maximum - base) as f32; + let mut state = floor + 1; + while state < 14 { + let frequency = states[state].0 as u64 * 1_000_000; + values[state] = (((frequency - base) as f32 * 100.0) / denominator).to_bits(); + state += 1; + } + values +} + +fn encode_m5_aux_perf_states(board: &Board, out: &mut [u8]) { + let table = &mut out[M5_HW_AUX_PERF_STATE.. + M5_HW_AUX_PERF_STATE + M5_HW_AUX_PERF_STATE_SIZE]; + table.fill(0); + put_u32(table, 0, 13); + put_u32(table, 4, 14); + for (state, &(frequency, core, sram)) in board.aux_perf_states.iter().enumerate() { + put_u32(table, 8 + state * 4, frequency); + for lane in 0..2 { + put_u32(table, 0x48 + state * 8 + lane * 4, core[lane]); + put_u32(table, 0xc8 + state * 8 + lane * 4, sram[lane]); + } + } +} + +pub(crate) const COMMON26_HW_PERF_INDEX_MAP: usize = 0x1ae4; + +pub(crate) const COMMON26_KSM_COMPLETION_TABLE_OFFSET: usize = 0x2734; + +pub(crate) const COMMON26_QOS_POINTER_OFFSET: usize = 0x27cc; + +pub(crate) const COMMON26_HW_FIXED_WORD_OFFSET: usize = 0x2714; + +pub(crate) const COMMON26_HW_FIXED_WORD_VALUE: u32 = 0x96; + +pub(crate) const J873_GPU_NUM_PERF_STATES: u32 = 13; + +const COMMON26_PERF_SCALE_VALUE: u32 = 0x3f82_8f5c; + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct RegisterWindowSpec { + pub(crate) slot: usize, + pub(crate) phys: u64, + + pub(crate) size_10: u32, + + pub(crate) size_14: u32, + + pub(crate) unk_18: u64, + + pub(crate) flag: u32, +} + +const fn spec(slot: usize, phys: u64, size: u32, unk_18: u64, flag: u32) -> RegisterWindowSpec { + RegisterWindowSpec { slot, phys, size_10: size, size_14: size, unk_18, flag } +} + +pub(crate) const COMMON26_REGION_C_ADT_WORDS: [(usize, u32); 3] = [ + (0x7c8, 1500), + (0x7e4, 0x3f4c_cccd), + (0x09c, J873_GPU_NUM_PERF_STATES * 100), +]; + +pub(crate) fn encode_common26_region_c(out: &mut [u8]) -> Result<(), InitdataError> { + if out.len() != REGION_C_SIZE { + return Err(InitdataError::BufferSize); + } + encode_region_c_for_profile(RootProfile::J813G17G25G76, out)?; + + let shifted = ABI_ALLOCATION_SIZES_25G76[ALLOC_REGION_C]; + if out[shifted..].iter().any(|byte| *byte != 0) { + return Err(InitdataError::BufferSize); + } + out.copy_within(0x38..shifted, 0x3c); + out[0x38..0x3c].fill(0); + for (offset, value) in COMMON26_REGION_C_ADT_WORDS { + put_u32(out, offset, value); + } + Ok(()) +} + +// J714 descriptors carry device-specific AON table indices. +const J714_AON_RECORDS: [[u32; 3]; 3] = [ + [0x6730_5452, 0x0079_0001, 0x0000_1401], + [0x6731_5452, 0x00bc_0001, 0x0000_1401], + [0x7230_5452, 0x00ce_0001, 0x0000_1401], +]; + +// J716c's rows tagged RT0g, RT1g and RT0r. +const J716C_AON_RECORDS: [[u32; 3]; 3] = [ + [0x6730_5452, 0x0058_0001, 0x0000_1401], + [0x6731_5452, 0x00a3_0001, 0x0000_1401], + [0x7230_5452, 0x00b8_0001, 0x0000_1401], +]; + +// J775D: both dies' records, g0TR-g3TR and r0TR-r1TR. +const J775D_AON_RECORDS: [[u32; 3]; 6] = [ + [0x6730_5452, 0x019f_0001, 0x0000_1401], + [0x6731_5452, 0x01bb_0001, 0x0000_1401], + [0x6732_5452, 0x00a7_0001, 0x0000_1401], + [0x6733_5452, 0x00c3_0001, 0x0000_1401], + [0x7230_5452, 0x0113_0001, 0x0000_1401], + [0x7231_5452, 0x001b_0001, 0x0000_1401], +]; + +pub(crate) fn encode_common26_aon_ptd(board: &Board, out: &mut [u8]) -> Result<(), InitdataError> { + if out.len() != M5_AON_PTD_SIZE || board.aon_records.len() * 12 > out.len() { + return Err(InitdataError::BufferSize); + } + out.fill(0); + for (index, record) in board.aon_records.iter().enumerate() { + for (word, value) in record.iter().enumerate() { + put_u32(out, index * 12 + word * 4, *value); + } + } + Ok(()) +} + +pub(crate) const COMMON26_CONTROL_OPENING_OPCODE_PRIMARY: u32 = 0x15; + +pub(crate) const COMMON26_CONTROL_OPENING_OPCODE_SECONDARY: u32 = 0x2a; + +pub(crate) const M5_CONTROL_OPENING_OPCODE_PRIMARY: u32 = COMMON26_CONTROL_OPENING_OPCODE_PRIMARY; + +pub(crate) const M5_CONTROL_OPENING_OPCODE_SECONDARY: u32 = + COMMON26_CONTROL_OPENING_OPCODE_SECONDARY; + +pub(crate) const M5_INITDATA_VERSION_WORD: u64 = 0x0c8e_0339_8728_04c0; + +pub(crate) const M5_ROOT_SIZE: usize = 0xc8; + +pub(crate) const ABI_ALLOCATION_SIZES_26A428_M5: [usize; 24] = [ + 0x2850, 0x0c18, 0x1048, 0x0e10, 0x126b0, 0x0068, 0x0800, 0x0088, 0x1800, 0x04c0, 0x4000, 0x0030, + 0xe730, 0x0780, 0x0020, 0x01b0, 0x4800, 0x28800, 0x79800, 0x9000, 0x00a8, 0x0008, 0x1ca0, 0x00e4, +]; + +pub(crate) const M5_SHARED_LAYOUT: SharedClusterLayout = + shared_cluster_layout(&ABI_ALLOCATION_SIZES_26A428_M5); + +pub(crate) const M5_PRIVATE_LAYOUT: PrivateClusterLayout = + private_cluster_layout(&ABI_ALLOCATION_SIZES_26A428_M5); + +pub(crate) const M5_SHARED_CLUSTER_SIZE: usize = M5_SHARED_LAYOUT.total; + +pub(crate) const M5_PRIVATE_CLUSTER_SIZE: usize = M5_PRIVATE_LAYOUT.total; + +pub(crate) const M5_HW_DATA_SIZE: usize = ABI_ALLOCATION_SIZES_26A428_M5[ALLOC_HW_DATA]; + +pub(crate) const M5_KICK_SHARED_SIZE: usize = 0x2000; +pub(crate) const M5_KICK_SHARED_POINTER: usize = 0x27dc; + +pub(crate) fn set_m5_kick_shared(out: &mut [u8], address: u64) -> Result<(), InitdataError> { + if out.len() != M5_HW_DATA_SIZE { + return Err(InitdataError::BufferSize); + } + required_address(address)?; + let end = address.checked_add(M5_KICK_SHARED_SIZE as u64 - 1) + .ok_or(InitdataError::AddressOutOfRange)?; + required_address(end)?; + if address & (UAT_PAGE_SIZE as u64 - 1) != 0 { + return Err(InitdataError::AddressOutOfRange); + } + put_u64(out, M5_KICK_SHARED_POINTER, address); + Ok(()) +} + +pub(crate) const M5_POLICY_SIZE: usize = ABI_ALLOCATION_SIZES_26A428_M5[ALLOC_POLICY]; + +pub(crate) const M5_STATUS_CONFIG_SIZE: usize = + ABI_ALLOCATION_SIZES_26A428_M5[ALLOC_STATUS_CONFIG]; + +pub(crate) const M5_REGION_C_ALLOCATION: usize = + ABI_ALLOCATION_SIZES_26A428_M5[ALLOC_REGION_C]; + +pub(crate) const M5_AON_PTD_SIZE: usize = ABI_ALLOCATION_SIZES_26A428_M5[ALLOC_AON_PTD]; + +pub(crate) const M5_SHARED_PREFIX_SIZE: usize = M5_SHARED_LAYOUT.brn_table; + +pub(crate) const M5_BUNDLE_VIEW_OFFSETS: [usize; 5] = [ + M5_SHARED_LAYOUT.block_c18, + M5_SHARED_LAYOUT.block_1048, + M5_SHARED_LAYOUT.block_e10, + M5_SHARED_LAYOUT.block_68, + M5_SHARED_LAYOUT.block_800, +]; + +pub(crate) const M5_BUNDLE_VIEW_EXTENTS: [usize; 5] = COMMON26_BUNDLE_VIEW_EXTENTS; + +pub(crate) const M5_WORK_RING_BUNDLE_OFFSETS: [usize; WORK_CHANNEL_COUNT] = + work_ring_offsets(&M5_SHARED_LAYOUT); + +pub(crate) const M5_MAIN_BUNDLE_OFFSETS: [usize; 2] = M5_SHARED_LAYOUT.main_config; + +pub(crate) const M5_CONTROL_RING_BUNDLE_OFFSETS: [usize; 2] = + M5_SHARED_LAYOUT.control_ring; + +pub(crate) const M5_PRIVATE_STATUS_A: [usize; 2] = [ + M5_PRIVATE_LAYOUT.roles[0].status_a, + M5_PRIVATE_LAYOUT.roles[1].status_a, +]; + +pub(crate) const M5_PRIVATE_STATE_GRIDS: [usize; 2] = [ + M5_PRIVATE_LAYOUT.dm_ring_states[0], + M5_PRIVATE_LAYOUT.secondary_control_state - WORK_STATE_GRID_OFFSETS[CONTROL_CHANNEL_INDEX], +]; + +pub(crate) const M5_STATUS_CONFIG_IN_STATUS_B: usize = + M5_PRIVATE_LAYOUT.status_config - M5_PRIVATE_LAYOUT.status_b; + +pub(crate) const M5_STATUS_B_SPAN: usize = + M5_PRIVATE_LAYOUT.roles[0].status_a - M5_PRIVATE_LAYOUT.status_b; + +pub(crate) const M5_RECOVERY_STATE: usize = 0x4ae0; +/// The firmware's progress word in the primary status B, beside the recovery state: the +/// firmware moves it as it works, and a reader that finds it changed since its previous read +/// counts the GPU as progressing. +pub(crate) const M5_PROGRESS: usize = 0x4b00; +pub(crate) const M5_HOST_RECOVERY: usize = 0x4af0; +pub(crate) const M5_RECOVERY_INFO: usize = 0x4298; +pub(crate) const M5_RECOVERY_SLOT_MASK: usize = 0x4ab8; + +pub(crate) const M5_STATUS_CONFIG_LINK_WORD: usize = 0x774; + +pub(crate) const M5_STATUS_CONFIG_LINK_PTR: usize = 0x778; + +pub(crate) const M5_HW_DATA_INSERT: usize = 0x267c; + +pub(crate) const M5_HW_DATA_INSERT_LEN: usize = 8; + +/// What the M5 graph's HW data and power policy depend on per board. Every board is a T6050 +/// running the same firmware build; [`crate::g17::Config::m5`] names the board's. +pub(crate) struct Board { + /// GPU clusters. + pub(crate) clusters: u32, + /// Cores of all clusters. + pub(crate) gpu_cores: u32, + /// The core type and firmware revision the HW data names. + core_revision: (u32, u32), + /// Main-domain performance states. + pub(crate) perf_states: &'static [M5PerfState; 14], + /// Voltage tables in [`Board::perf_states`]. + perf_tables: usize, + /// Auxiliary-domain performance states. + aux_perf_states: &'static [M5AuxPerfState; 14], + /// The relative-frequency curve of [`Board::perf_states`]. + relative_frequency: [u32; 16], + /// Rows of the board's AON table the firmware reads. + aon_records: &'static [[u32; 3]], + /// Register windows the firmware maps, in device-VA order. + register_windows: &'static [M5RegisterWindowSpec], + /// The smallest parameter buffer, bytes. Renders carry it in pages too + /// (`target::min_parameter_buffer_pages()`). + min_parameter_buffer: u32, + /// Render block size and private-memory address inventory of the board's + /// render pools, bytes. + pub(crate) render_pool: (usize, u64), + /// The leakage calibration words [`Board::controller_defaults`] are valid for, or `None` + /// where they hold for every part of the board. + pub(crate) calibration: Option<[u32; 3]>, + /// The policy controllers' configuration, where the board has one. + pub(crate) controller_defaults: Option Result<(), InitdataError>>, + /// Region C's idle standby timer where the board's differs from the image's 1500 us, us. + pub(crate) idle_standby_us: Option, + /// The active memory cache controller mask, where it differs from the inherited 0xf. + mcc_mask: Option, + /// The two topology words (16, and the core count), where the board sets them. + topology_words: Option<(u32, u32)>, + /// The hotspot at which the thermal governor's fan curve starts, millidegrees C. + pub(crate) fan_curve_start_mc: i32, + /// Whether the host encodings are the G17D ones measured on the M5 Ultra + /// ([`Board::g17d_forms`]). + g17d_forms: bool, + /// Whether explicit waits on other kicks take the completion class + /// ([`Board::completion_inputs`]). + completion_inputs: bool, +} + +/// M5 Pro: two clusters of ten cores. +pub(crate) const J714: Board = Board { + clusters: 2, + gpu_cores: 20, + core_revision: (0x18, 4), + perf_states: &J714_GPU_PERF_STATES, + perf_tables: 2, + aux_perf_states: &J714_AUX_PERF_STATES, + relative_frequency: relative_frequency(&J714_GPU_PERF_STATES, 1), + aon_records: &J714_AON_RECORDS, + register_windows: &J714_REGISTER_WINDOWS, + min_parameter_buffer: 0x0020_0000, + render_pool: (0x40_0000, 1 << 32), + calibration: Some([0xd2035701, 0xac3a1280, 0x0a281d81]), + controller_defaults: Some(encode_m5_policy_controller_defaults), + idle_standby_us: None, + mcc_mask: None, + topology_words: None, + fan_curve_start_mc: 84_000, + g17d_forms: false, + completion_inputs: true, +}; + +/// M5 Max: four clusters of ten cores. +pub(crate) const J716C: Board = Board { + clusters: 4, + gpu_cores: 40, + core_revision: (0x19, 4), + perf_states: &J716C_GPU_PERF_STATES, + perf_tables: 4, + aux_perf_states: &J716C_AUX_PERF_STATES, + relative_frequency: relative_frequency(&J716C_GPU_PERF_STATES, 1), + aon_records: &J716C_AON_RECORDS, + register_windows: &J716C_REGISTER_WINDOWS, + min_parameter_buffer: 0x0030_0000, + render_pool: (0x80_0000, 1 << 33), + // Its own controller configuration, for a part with this calibration. + calibration: Some([0x7e4a4902, 0x8c4f3362, 0x96be56e5]), + controller_defaults: Some(encode_m5_policy_controller_defaults_j716c), + idle_standby_us: None, + // The ADT GPU node's active-mcc-bit-vector: eight memory cache controllers. + mcc_mask: Some(0xff), + topology_words: Some((16, 40)), + fan_curve_start_mc: 84_000, + g17d_forms: false, + completion_inputs: false, +}; + +/// J775D's performance controllers run from P3. +const J775D_PERF_FLOOR: usize = 3; + +/// M5 Ultra: two dies of four clusters of ten cores (G17D). Values from the J775D ADT. +pub(crate) const J775D: Board = Board { + clusters: 8, + gpu_cores: 80, + // Firmware revision 2 for raw revision 0x11. + core_revision: (0x1a, 2), + perf_states: &J775D_GPU_PERF_STATES, + // One voltage table per cluster. + perf_tables: 8, + aux_perf_states: &J775D_AUX_PERF_STATES, + relative_frequency: relative_frequency(&J775D_GPU_PERF_STATES, J775D_PERF_FLOOR), + aon_records: &J775D_AON_RECORDS, + register_windows: &J775D_REGISTER_WINDOWS, + // The minimum parameter buffer for eight clusters. + min_parameter_buffer: 0x0050_0000, + // 16 MiB render blocks in a 16 GiB inventory. + render_pool: (0x100_0000, 1 << 34), + // The power model's synthetic leakage inputs hold for every J775D: no calibration words. + calibration: None, + controller_defaults: Some(encode_m5_policy_controller_defaults_j775d), + idle_standby_us: Some(3500), + // The ADT GPU node's active-mcc-bit-vector: eight memory cache controllers. + mcc_mask: Some(0xff), + topology_words: None, + // The SMC raises the fans in its automatic mode from a 97 C hotspot under a sustained + // full-chip load, with the CPU and GPU still at their top states: the curve starts there, + // above the SMC's own start rather than below it. + fan_curve_start_mc: 97_000, + g17d_forms: true, + completion_inputs: false, +}; + +const _: () = { + use super::target::min_parameter_buffer_pages; + assert!(J714.min_parameter_buffer as u64 == min_parameter_buffer_pages(J714.clusters) << 15); + assert!(J716C.min_parameter_buffer as u64 == min_parameter_buffer_pages(J716C.clusters) << 15); + assert!(J775D.min_parameter_buffer as u64 == min_parameter_buffer_pages(J775D.clusters) << 15); +}; + +/// Populate the enabled-USC count from the device's sampled core masks. +/// This count covers the whole device, not one cluster. +pub(crate) fn set_m5_enabled_cores(out: &mut [u8], enabled: u32) -> Result<(), InitdataError> { + if out.len() != M5_HW_DATA_SIZE { + return Err(InitdataError::BufferSize); + } + if enabled == 0 || enabled > 128 { + return Err(InitdataError::InterfaceMismatch); + } + put_u32(out, 0x10a4, enabled); + put_u32(out, 0x268c, enabled); + Ok(()) +} + +pub(crate) const M5_POLICY_STATE_PTR: usize = 0x8254; + +pub(crate) const M5_POLICY_AON_PTD_PTR: usize = 0x11a1c; + +pub(crate) const J714_GPU_POWER_SAMPLE_PERIOD_AIC_CLOCKS: u32 = 200_000; + +pub(crate) const J714_GPU_PERF_BASE_PSTATE: u32 = 1; + +const J714_POWER_POLICY_MINIMUM_TABLE: usize = 0x1cab; + +const J714_POWER_POLICY_MINIMUM_ENTRIES: usize = 16 * 32; + +pub(crate) const J714_GPU_NUM_PERF_STATES: u32 = 13; + +/// A main-domain performance state: MHz, then core and SRAM mV for each voltage table. HW perf +/// lane n carries table n; lanes past the board's table count carry table 0. +pub(crate) type M5PerfState = (u32, [u32; M5_PERF_LANES], [u32; M5_PERF_LANES]); + +/// Voltage tables an [`M5PerfState`] can hold. +pub(crate) const M5_PERF_LANES: usize = 8; + +/// Two-table states (MHz, two core-voltage tables, two SRAM-voltage tables), the rest 0. +const fn two_tables(states: [(u32, u32, u32, u32, u32); 14]) -> [M5PerfState; 14] { + let mut out = [(0, [0; M5_PERF_LANES], [0; M5_PERF_LANES]); 14]; + let mut state = 0; + while state < 14 { + let (frequency, core0, core1, sram0, sram1) = states[state]; + out[state].0 = frequency; + out[state].1[0] = core0; + out[state].1[1] = core1; + out[state].2[0] = sram0; + out[state].2[1] = sram1; + state += 1; + } + out +} + +// Frequency in MHz, two core-voltage tables, two SRAM-voltage tables (mV). +pub(crate) const J714_GPU_PERF_STATES: [M5PerfState; 14] = two_tables([ + (0, 125, 125, 775, 775), (338, 605, 605, 775, 775), + (486, 620, 635, 775, 775), (636, 655, 655, 775, 775), + (796, 705, 705, 775, 775), (888, 715, 715, 775, 775), + (988, 745, 745, 775, 775), (1084, 770, 770, 775, 775), + (1182, 800, 800, 800, 800), (1278, 840, 840, 840, 840), + (1374, 880, 880, 880, 880), (1470, 925, 925, 925, 925), + (1578, 1000, 1000, 1000, 1000), (1620, 1025, 1040, 1025, 1040), +]); + +/// Four-table states (MHz, four core-voltage tables, four SRAM-voltage tables), the rest 0. +const fn four_tables(states: [(u32, [u32; 4], [u32; 4]); 14]) -> [M5PerfState; 14] { + let mut out = [(0, [0; M5_PERF_LANES], [0; M5_PERF_LANES]); 14]; + let mut state = 0; + while state < 14 { + let (frequency, core, sram) = states[state]; + out[state].0 = frequency; + let mut table = 0; + while table < 4 { + out[state].1[table] = core[table]; + out[state].2[table] = sram[table]; + table += 1; + } + state += 1; + } + out +} + +// J716c ADT perf-states / perf-states-sram: four tables (0 == 3, 1 == 2). +pub(crate) const J716C_GPU_PERF_STATES: [M5PerfState; 14] = four_tables([ + (0, [125, 125, 125, 125], [775, 775, 775, 775]), + (338, [635, 635, 635, 635], [775, 775, 775, 775]), + (486, [655, 640, 640, 655], [775, 775, 775, 775]), + (636, [680, 665, 665, 680], [775, 775, 775, 775]), + (796, [730, 715, 715, 730], [775, 775, 775, 775]), + (888, [740, 725, 725, 740], [775, 775, 775, 775]), + (988, [775, 760, 760, 775], [775, 775, 775, 775]), + (1084, [785, 785, 785, 785], [785, 785, 785, 785]), + (1182, [805, 810, 810, 805], [805, 810, 810, 805]), + (1278, [840, 840, 840, 840], [840, 840, 840, 840]), + (1374, [875, 865, 865, 875], [875, 865, 865, 875]), + (1470, [920, 905, 905, 920], [920, 905, 905, 920]), + (1578, [995, 980, 980, 995], [995, 980, 980, 995]), + (1620, [1015, 1015, 1015, 1015], [1015, 1015, 1015, 1015]), +]); + +// J775D ADT sgx perf-states (one table per cluster) and perf-states-sram, mV. +pub(crate) const J775D_GPU_PERF_STATES: [M5PerfState; 14] = [ + (0, [125, 125, 125, 125, 125, 125, 125, 125], [775, 775, 775, 775, 775, 775, 775, 775]), + (338, [615, 615, 615, 615, 615, 610, 610, 615], [775, 775, 775, 775, 775, 775, 775, 775]), + (486, [670, 655, 655, 670, 650, 665, 665, 650], [775, 775, 775, 775, 775, 775, 775, 775]), + (636, [695, 690, 690, 695, 690, 690, 690, 690], [775, 775, 775, 775, 775, 775, 775, 775]), + (796, [760, 745, 745, 760, 745, 740, 740, 745], [775, 775, 775, 775, 775, 775, 775, 775]), + (888, [770, 755, 755, 770, 750, 750, 750, 750], [775, 775, 775, 775, 775, 775, 775, 775]), + (988, [795, 780, 780, 795, 775, 775, 775, 775], [795, 780, 780, 795, 775, 775, 775, 775]), + (1084, [805, 805, 805, 805, 815, 800, 800, 815], [805, 805, 805, 805, 815, 800, 800, 815]), + (1182, [815, 810, 810, 815, 825, 820, 820, 825], [815, 810, 810, 815, 825, 820, 820, 825]), + (1278, [860, 845, 845, 860, 870, 855, 855, 870], [860, 845, 845, 860, 870, 855, 855, 870]), + (1374, [885, 885, 885, 885, 895, 895, 895, 895], [885, 885, 885, 885, 895, 895, 895, 895]), + (1470, [930, 930, 930, 930, 940, 935, 935, 940], [930, 930, 930, 930, 940, 935, 935, 940]), + (1578, [1020, 1005, 1005, 1020, 1030, 1015, 1015, 1030], + [1020, 1005, 1005, 1020, 1030, 1015, 1015, 1030]), + (1620, [1045, 1040, 1040, 1045, 1065, 1050, 1050, 1065], + [1045, 1040, 1040, 1045, 1065, 1050, 1050, 1065]), +]; + +pub(crate) const J714_GPU_BASE: u64 = 0x23_0000_0000; + +#[derive(Copy, Clone)] +pub(crate) struct M5RegisterWindowSpec { + pub(crate) slot: usize, + + pub(crate) instances: &'static [u64], + pub(crate) size: u32, + + pub(crate) unk_18: u64, + pub(crate) flag: u32, +} + +const fn jw(slot: usize, instances: &'static [u64], size: u32, unk_18: u64, flag: u32) + -> M5RegisterWindowSpec { + M5RegisterWindowSpec { slot, instances, size, unk_18, flag } +} + +const G: u64 = J714_GPU_BASE; + +const J714_REGISTER_WINDOW_SPECS: [M5RegisterWindowSpec; 29] = [ + jw(0, &[0x2_8041_4000], 0x4000, 0, 2), + jw(3, &[0x22_2010_4000, 0x22_2210_4000, 0x22_2410_4000, 0x22_2610_4000], 0x18000, 0, 2), + jw(4, &[0x2_8046_8000], 0x4000, 0, 0), + jw(9, &[0x2_8425_0000], 0x4000, 0, 2), + jw(10, &[0x2_8424_0000], 0x4000, 0, 0), + jw(11, &[0x2_8425_8000], 0x4000, 0, 2), + jw(12, &[0x5_0965_c000], 0x4000, 0, 2), + jw(14, &[0x23_8030_0000], 0x8000, 0, 0), + jw(15, &[0x2_8cdc_0000], 0x24000, 0, 2), + jw(17, &[G], 0x21500, 0, 2), + jw(18, &[G + 0xeb_4000], 0x4000, 0, 2), + jw(20, &[0x23_0300_0000, 0x23_0320_0000], 0x40000, 0, 2), + jw(22, &[G + 0x100_0000], 0x8000, 0, 2), + jw(23, &[G + 0xf0_0000], 0x10000, 0, 2), + jw(24, &[G + 0xec_0000, G + 0xed_0000], 0x10000, 0, 2), + jw(31, &[G + 0xd0_4000], 0x8000, 0xd0_4000, 2), + jw(33, &[G + 0xd5_0000], 0x10000, 0xd5_0000, 2), + jw(34, &[G + 0xd1_0000], 0x4000, 0xd1_0000, 2), + jw(36, &[G + 0xd4_0000], 0x4000, 0xd4_0000, 2), + jw(37, &[G + 0xd6_0000], 0x20000, 0xd6_0000, 2), + jw(38, &[G + 0xd4_4000], 0x4000, 0xd4_4000, 2), + jw(39, &[G + 0xd4_c000], 0x200, 0xd4_c000, 2), + jw(40, &[G + 0xe0_0000], 0x4000, 0xe0_0000, 2), + jw(42, &[G + 0xe4_0000], 0x4000, 0xe4_0000, 2), + jw(44, &[G + 0xe0_8000, G + 0xe1_0000], 0x8000, 0, 2), + jw(46, &[G + 0xe3_3800], 0x4000, 0, 2), + jw(48, &[G + 0xe6_0000], 0x58, 0xe6_0000, 2), + jw(52, &[G + 0xe4_8000], 0x8000, 0, 2), + jw(53, &[G + 0x2_3d00], 0x200, 0x2_3d00, 2), +]; + +const J714_REGISTER_WINDOW_58: M5RegisterWindowSpec = + jw(58, &[G + 0xe7_3800], 0x4000, 0, 2); + +/// J714's windows, ending with slot 58. +const J714_REGISTER_WINDOWS: [M5RegisterWindowSpec; 30] = { + let mut windows = [J714_REGISTER_WINDOW_58; 30]; + let mut index = 0; + while index < J714_REGISTER_WINDOW_SPECS.len() { + windows[index] = J714_REGISTER_WINDOW_SPECS[index]; + index += 1; + } + windows +}; + +/// J714 with all eight AMCCs in slot 3 (J716c amcc-enable-mask 0xff, MCC +/// reg[14..21] + 0x104000) and four instances, one per cluster, in slots 20 and +/// 24, where J714 has two at the same stride. Slot 44 is a fixed pair on every M5 +/// part. +const J716C_REGISTER_WINDOW_SPECS: [M5RegisterWindowSpec; 29] = { + let mut specs = J714_REGISTER_WINDOW_SPECS; + assert!(specs[1].slot == 3 && specs[11].slot == 20 && specs[14].slot == 24); + specs[1] = jw(3, &[0x22_2010_4000, 0x22_2210_4000, 0x22_2410_4000, 0x22_2610_4000, + 0x22_2810_4000, 0x22_2a10_4000, 0x22_2c10_4000, 0x22_2e10_4000], + 0x18000, 0, 2); + specs[11] = jw(20, &[0x23_0300_0000, 0x23_0320_0000, 0x23_0340_0000, 0x23_0360_0000], + 0x40000, 0, 2); + specs[14] = jw(24, &[G + 0xec_0000, G + 0xed_0000, G + 0xee_0000, G + 0xef_0000], + 0x10000, 0, 2); + specs +}; + +/// J716c's windows, ending with slot 58. +const J716C_REGISTER_WINDOWS: [M5RegisterWindowSpec; 30] = { + let mut windows = [J714_REGISTER_WINDOW_58; 30]; + let mut index = 0; + while index < J716C_REGISTER_WINDOW_SPECS.len() { + windows[index] = J716C_REGISTER_WINDOW_SPECS[index]; + index += 1; + } + windows +}; + +/// Second die of J775D. +const D: u64 = 0x40_0000_0000; + +/// J775D: the explicit instances of both dies (ADT), in device-VA order. +const J775D_REGISTER_WINDOWS: [M5RegisterWindowSpec; 32] = [ + jw(0, &[0x2_8041_4000], 0x4000, 0, 2), + // One per active memory cache controller. + jw(3, &[0x22_2010_4000, 0x22_2210_4000, 0x22_2410_4000, 0x22_2610_4000, + 0x22_2810_4000, 0x22_2a10_4000, 0x22_2c10_4000, 0x22_2e10_4000], 0x18000, 0, 2), + jw(4, &[0x2_8046_8000], 0x4000, 0, 0), + jw(9, &[0x2_8425_0000], 0x4000, 0, 2), + jw(10, &[0x2_8424_0000], 0x4000, 0, 0), + jw(11, &[0x2_8425_8000], 0x4000, 0, 2), + jw(12, &[0x5_0965_c000], 0x4000, 0, 2), + jw(13, &[0x45_0965_c000], 0x4000, 0, 2), + jw(14, &[0x23_8030_0000], 0x8000, 0, 0), + jw(15, &[0x2_8cdc_0000], 0x24000, 0, 2), + jw(17, &[G], 0x21500, 0, 2), + jw(18, &[G + 0xeb_4000, G + 0xeb_4000 + D], 0x4000, 0, 2), + jw(20, &[G + 0x300_0000, G + 0x320_0000, G + 0x340_0000, G + 0x360_0000, + G + 0x380_0000, G + 0x3a0_0000, G + 0x3c0_0000, G + 0x3e0_0000, + 0x23_035a_8000, 0x23_037a_8000, 0x23_03da_8000, 0x23_03fa_8000], 0x40000, 0, 2), + jw(22, &[G + 0x100_0000], 0x8000, 0, 2), + jw(23, &[G + 0xf0_0000, G + 0xf0_0000 + D], 0x10000, 0, 2), + jw(24, &[G + 0xec_0000, G + 0xed_0000, G + 0xee_0000, G + 0xef_0000, + G + 0xec_0000 + D, G + 0xed_0000 + D, G + 0xee_0000 + D, G + 0xef_0000 + D], + 0x10000, 0, 2), + jw(31, &[G + 0xd0_4000], 0x8000, 0xd0_4000, 2), + jw(33, &[G + 0xd5_0000], 0x10000, 0xd5_0000, 2), + jw(34, &[G + 0xd1_0000], 0x4000, 0xd1_0000, 2), + jw(36, &[G + 0xd4_0000], 0x4000, 0xd4_0000, 2), + jw(37, &[G + 0xd6_0000], 0x20000, 0xd6_0000, 2), + jw(38, &[G + 0xd4_4000], 0x4000, 0xd4_4000, 2), + jw(39, &[G + 0xd4_c000], 0x200, 0xd4_c000, 2), + jw(40, &[G + 0xe0_0000], 0x4000, 0xe0_0000, 2), + jw(42, &[G + 0xe4_0000], 0x4000, 0xe4_0000, 2), + jw(44, &[G + 0xe0_8000, G + 0xe1_0000], 0x8000, 0, 2), + jw(46, &[G + 0xe3_3800], 0x4000, 0, 2), + jw(48, &[G + 0xe6_0000], 0x58, 0xe6_0000, 2), + jw(52, &[G + 0xe4_8000], 0x8000, 0, 2), + jw(53, &[G + 0x2_3d00], 0x200, 0x2_3d00, 2), + jw(58, &[G + 0xe7_3800], 0x4000, 0, 2), + jw(59, &[0x23_0358_0000, 0x23_0378_0000, 0x23_03d8_0000, 0x23_03f8_0000], 0x60000, 0, 2), +]; + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct M5RegisterWindow { + pub(crate) slot: usize, + pub(crate) device_va: u64, + pub(crate) count: usize, + pub(crate) size: u32, + pub(crate) unk_18: u64, + pub(crate) flag: u32, +} + +/// The most register windows of any board. +pub(crate) const M5_MAX_REGISTER_WINDOWS: usize = J775D_REGISTER_WINDOWS.len(); + +impl Board { + /// Whether the board takes the host encodings measured on the M5 Ultra (G17D) where they + /// differ from J714's, as `super::compute` and the render registers read them. + pub(crate) const fn g17d_forms(&self) -> bool { + self.g17d_forms + } + + /// Whether a kick's explicit waits on other kicks (inputs from other queues, other-engine + /// producers) take the completion class, which holds them in firmware until those kicks + /// complete. A wait on another queue's compute stays in firmware only for work that declares + /// a visible completion (DRM_ASAHI_COMPUTE_VISIBLE_COMPLETION); others wait on the host. + pub(crate) const fn completion_inputs(&self) -> bool { + self.completion_inputs + } + + /// The board's `index`th register window. + pub(crate) fn register_window_spec(&self, index: usize) -> Option<&'static M5RegisterWindowSpec> { + self.register_windows.get(index) + } + + /// The board's register windows: the first [`Board::register_window_count`] entries. + pub(crate) fn register_windows(&self) -> [M5RegisterWindow; M5_MAX_REGISTER_WINDOWS] { + m5_register_windows(self.register_windows) + } + + /// Register windows of the board. + pub(crate) fn register_window_count(&self) -> usize { + self.register_windows.len() + } +} + +fn m5_register_windows(specs: &'static [M5RegisterWindowSpec]) + -> [M5RegisterWindow; M5_MAX_REGISTER_WINDOWS] +{ + let mask = UAT_PAGE_SIZE as u64 - 1; + let mut out = [M5RegisterWindow { slot: 0, device_va: 0, count: 0, size: 0, unk_18: 0, flag: 0 }; + M5_MAX_REGISTER_WINDOWS]; + let mut next: u64 = 0; + for (window, spec) in out.iter_mut().zip(specs) { + let va = next + (spec.instances[0] & mask); + let span = spec.instances.len() as u64 * ((spec.size as u64 + mask) & !mask); + *window = M5RegisterWindow { + slot: spec.slot, + device_va: HW_REGISTER_APERTURE.start + va, + count: spec.instances.len(), + size: spec.size, + unk_18: spec.unk_18, + flag: spec.flag, + }; + let end = (va + span + mask) & !mask; + next = (end + UAT_PAGE_SIZE as u64 + 0x7fff) & !0x7fff; + } + out +} + +pub(crate) fn encode_m5_register_map( + board: &Board, + windows: &[M5RegisterWindow], + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != COMMON26_HW_REGISTER_MAP_SLOTS * HW_REGISTER_MAP_ENTRY_SIZE { + return Err(InitdataError::BufferSize); + } + out.fill(0); + for (index, window) in windows.iter().enumerate() { + let spec = board.register_windows.get(index).ok_or(InitdataError::RegisterWindow)?; + if window.slot >= COMMON26_HW_REGISTER_MAP_SLOTS + || !HW_REGISTER_APERTURE.contains(&window.device_va) + { + return Err(InitdataError::RegisterWindow); + } + let base = window.slot * HW_REGISTER_MAP_ENTRY_SIZE; + put_u64(out, base, spec.instances[0]); + put_u64(out, base + 0x08, window.device_va); + put_u32(out, base + 0x10, window.count as u32 * window.size); + put_u32(out, base + 0x14, window.size); + put_u64(out, base + 0x18, window.unk_18); + put_u32(out, base + 0x20, window.flag); + } + Ok(()) +} + +pub(crate) fn encode_m5_hw_data( + board: &Board, + aliases: [G17PKsmCompletionAliases; M5_25G76_COMPLETION_QUEUE_COUNT], + qos_high: u64, + windows: &[M5RegisterWindow], + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != M5_HW_DATA_SIZE { + return Err(InitdataError::BufferSize); + } + required_address(qos_high)?; + let qos_end = qos_high + .checked_add(G17P_QOS_RESOURCE_SIZE as u64) + .ok_or(InitdataError::AddressOutOfRange)?; + required_address(qos_end - 1)?; + let mut completion = [0; M5_25G76_COMPLETION_QUEUE_COUNT * G17P_KSM_COMPLETION_DESCRIPTOR_STRIDE]; + encode_m5_25g76_completion_table(aliases, &mut completion)?; + for alias in aliases { + if qos_high < alias.high + M5_25G76_COMPLETION_BACKING_SIZE as u64 && alias.high < qos_end { + return Err(InitdataError::PairAliasedObject); + } + } + + let g18 = &mut out[..COMMON26_HW_DATA_SIZE]; + g18.fill(0); + write_relocated_words(M5_25G76_HW_PLATFORM_WORDS, 0, COMMON26_HW_DATA_BANDS, g18)?; + encode_m5_register_map(board, windows, + &mut g18[HW_REGISTER_MAP_OFFSET..COMMON26_HW_REGISTER_MAP_END])?; + put_u32(g18, COMMON26_HW_CHIP_ID_OFFSET, 0x6050); + put_u32(g18, COMMON26_HW_PERF_STATE_COUNT, J714_GPU_NUM_PERF_STATES); + put_u32(g18, COMMON26_HW_FIXED_WORD_OFFSET, COMMON26_HW_FIXED_WORD_VALUE); + + put_u32(g18, 0x0fb4, 1); + // J714's 26A428 HW data carries 1 here. The firmware copies this + // header at initialization. + put_u32(g18, 0x0fb8, 1); + let (core_type, revision) = board.core_revision; + put_u32(g18, 0x10d4, core_type); + put_u32(g18, 0x10d8, revision); + put_u32(g18, 0x10dc, board.gpu_cores); + put_u32(g18, 0x283a, 1); + put_u32(g18, 0x283e, 1); + let states = board.perf_states; + for (state, &(frequency, core, sram)) in states.iter().enumerate() { + put_u32(g18, COMMON26_HW_PERF_FREQUENCY + state * 4, frequency); + put_u32(g18, COMMON26_HW_PERF_SCALE + state * 4, COMMON26_PERF_SCALE_VALUE); + + let relative = frequency * 100 / states[13].0; + put_u32(g18, COMMON26_HW_PERF_RELATIVE_A + state * 4, relative); + for lane in 0..COMMON26_HW_PERF_LANES { + let row = state * COMMON26_HW_PERF_ROW_STRIDE + lane * 4; + let table = if lane < board.perf_tables { lane } else { 0 }; + put_u32(g18, COMMON26_HW_PERF_CORE_VOLTAGE + row, core[table]); + put_u32(g18, COMMON26_HW_PERF_SRAM_VOLTAGE + row, sram[table]); + } + } + for index in 0..COMMON26_HW_PERF_ROWS { + put_u32(g18, COMMON26_HW_PERF_INDEX_MAP + index * 4, index as u32); + put_u32(g18, COMMON26_HW_PERF_RELATIVE_B + index * 4, board.relative_frequency[index]); + put_u32(g18, M5_HW_AUX_RELATIVE_FREQUENCY + index * 4, board.relative_frequency[index]); + } + encode_m5_aux_perf_states(board, g18); + let table = COMMON26_KSM_COMPLETION_TABLE_OFFSET; + g18[table..table + completion.len()].copy_from_slice(&completion); + put_u64(g18, COMMON26_QOS_POINTER_OFFSET, qos_high); + + let insert = M5_HW_DATA_INSERT; + let len = M5_HW_DATA_INSERT_LEN; + out.copy_within(insert..COMMON26_HW_DATA_SIZE, insert + len); + out[insert..insert + len].fill(0); + put_u32(out, 0x2660, 0); + put_u32(out, 0x2728, 0); + put_u32(out, 0x272c, 1); + if let Some(mask) = board.mcc_mask { + put_u64(out, 0x10a8, mask); + } + if let Some((word, cores)) = board.topology_words { + put_u32(out, 0x100c, word); + put_u32(out, 0x1048, cores); + } + // The inherited J813 minimum parameter-buffer size is replaced per board. + put_u32(out, 0x26b0, board.min_parameter_buffer); + Ok(()) +} + +pub(crate) const M5_POLICY_MAXIMUM_POWER: usize = 0xa18; + +/// Region C's idle standby timer: how long the idle GPU waits before it goes to standby, us. +pub(crate) fn set_m5_region_idle_standby(out: &mut [u8], microseconds: u32) + -> Result<(), InitdataError> { + if out.len() < M5_REGION_C_ALLOCATION { return Err(InitdataError::BufferSize); } + if microseconds == 0 { return Err(InitdataError::InterfaceMismatch); } + put_u32(out, 0x7c8, microseconds); + Ok(()) +} + +/// Apply the calibrated maximum shared by the policy and host power input. +pub(crate) fn set_m5_region_maximum_power(out: &mut [u8], maximum: u32) + -> Result<(), InitdataError> { + if out.len() < M5_REGION_C_ALLOCATION { return Err(InitdataError::BufferSize); } + if maximum == 0 { return Err(InitdataError::InterfaceMismatch); } + put_u32(out, 0x88, maximum); + Ok(()) +} + +pub(crate) fn encode_m5_region_c(out: &mut [u8]) -> Result<(), InitdataError> { + if out.len() < M5_REGION_C_ALLOCATION { + return Err(InitdataError::BufferSize); + } + out.fill(0); + encode_common26_region_c(&mut out[..REGION_C_SIZE])?; + + out.copy_within(0xe10..0xe78, 0x1c34); + out[0x9e4..0x1c34].fill(0); + out.copy_within(0x7f4..0x9e4, 0x7f8); + put_u32(out, 0x7f4, 1); + + put_u32(out, 0x9e0, 50); + + put_u32(out, 0x08c, J714_GPU_NUM_PERF_STATES * 100); + put_u32(out, 0x090, J714_GPU_NUM_PERF_STATES * 100); + put_u32(out, 0x09c, (J714_GPU_NUM_PERF_STATES + 1) * 100); + put_u32(out, 0x034, 1); + put_u32(out, 0x038, 1); + put_u32(out, 0x7e4, 0x3f19_999a); + put_u32(out, 0x1c30, 250); + const FW_UTIL_TUNABLES: [u8; 32] = [ + 1, 0, 0, 0, 0, 2, 2, 1, 1, 40, 10, 1, 1, 1, 2, 40, + 10, 1, 1, 1, 2, 40, 10, 1, 1, 1, 1, 40, 10, 1, 1, 0, + ]; + out[0x9e8..0xa08].copy_from_slice(&FW_UTIL_TUNABLES); + Ok(()) +} + +pub(crate) fn encode_m5_primary_status_b( + fwctl_state: u64, + fwctl_ring: u64, + status_b: u64, + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != M5_STATUS_B_SPAN { + return Err(InitdataError::BufferSize); + } + required_address(fwctl_state)?; + required_address(fwctl_ring)?; + required_address(status_b)?; + let config = M5_STATUS_CONFIG_IN_STATUS_B; + out.fill(0); + encode_status_block(&mut out[..STATUS_BLOCK_SIZE])?; + put_u64(out, COMMON26_STATUS_B_FWCTL_STATE, fwctl_state); + put_u64(out, COMMON26_STATUS_B_FWCTL_RING, fwctl_ring); + put_u32(out, COMMON26_STATUS_B_CONFIG_HEADER, 1); + put_u32(out, COMMON26_STATUS_B_LINK_WORD, 0); + put_u64(out, COMMON26_STATUS_B_LINK_PTR, + status_b + (config + M5_STATUS_CONFIG_LINK_WORD) as u64); + write_relocated_words(M5_25G76_STATUS_CONFIG_WORDS, 0, COMMON26_STATUS_CONFIG_BANDS, + &mut out[config..config + M5_STATUS_CONFIG_SIZE])?; + put_u32(out, config + M5_STATUS_CONFIG_LINK_WORD, 1); + put_u64(out, config + M5_STATUS_CONFIG_LINK_PTR, + status_b + COMMON26_STATUS_B_LINK_WORD as u64); + Ok(()) +} + +pub(crate) fn encode_m5_private_cluster( + private_base: u64, + fwctl_state: u64, + out: &mut [u8], +) -> Result<(), InitdataError> { + let layout = M5_PRIVATE_LAYOUT; + if out.len() != layout.total { + return Err(InitdataError::BufferSize); + } + required_address(private_base)?; + let private_end = private_base + .checked_add(layout.total as u64) + .ok_or(InitdataError::AddressOutOfRange)?; + required_address(private_end - 1)?; + required_address(fwctl_state)?; + let fwctl_ring = fwctl_state + .checked_add(COMMON26_FWCTL_RING_OFFSET as u64) + .ok_or(InitdataError::AddressOutOfRange)?; + required_address(fwctl_ring)?; + if fwctl_state < private_end && private_base < fwctl_ring + CONTROL_RECORD_SIZE as u64 { + return Err(InitdataError::PairAliasedObject); + } + out.fill(0); + let status_b = layout.status_b; + let status_a = layout.roles[0].status_a; + encode_m5_primary_status_b(fwctl_state, fwctl_ring, private_base + status_b as u64, + &mut out[status_b..status_a])?; + for role in layout.roles { + encode_status_block(&mut out[role.status_a..role.status_a + STATUS_BLOCK_SIZE])?; + } + + let extra = layout.secondary_extra_1; + encode_secondary_root_extra_1_for_profile(RootProfile::J873G18G26A428, + &mut out[extra..extra + STATUS_BLOCK_SIZE])?; + put_u32(out, layout.control_state + CHANNEL_STATE_PRODUCER * CHANNEL_STATE_SPACING, + CONTROL_OPENING_PRIMARY_PRODUCER); + put_u32(out, layout.secondary_control_state + CHANNEL_STATE_PRODUCER * CHANNEL_STATE_SPACING, + CONTROL_OPENING_SECONDARY_PRODUCER); + Ok(()) +} + +pub(crate) fn encode_m5_bundle_policy( + bindings: M5PolicyBindings, + out: &mut [u8], +) -> Result<(), InitdataError> { + let layout = M5_SHARED_LAYOUT; + if out.len() < M5_SHARED_PREFIX_SIZE { + return Err(InitdataError::BufferSize); + } + required_address(bindings.state)?; + required_address(bindings.aon_ptd)?; + out[layout.block_c18..M5_SHARED_PREFIX_SIZE].fill(0); + let m5 = M5_25G76_SHARED_LAYOUT; + for &(offset, value) in M5_25G76_POLICY_WORDS { + if (m5.block_1048..m5.block_e10).contains(&offset) { + put_u32(out, layout.block_1048 + (offset - m5.block_1048), value); + } + } + let policy = &mut out[layout.policy..layout.policy + M5_POLICY_SIZE]; + put_u32(policy, 0x04, J714_GPU_POWER_SAMPLE_PERIOD_AIC_CLOCKS); + put_u32(policy, 0x08, J714_GPU_POWER_SAMPLE_PERIOD_AIC_CLOCKS); + + for (offset, value) in [ + (0x10, 4), (0x14, 0x3f80_0000), (0x2c, 1), (0x30, 1), + (0x40, J714_GPU_PERF_BASE_PSTATE * 100), (0x44, 1), + (0x48, J714_GPU_NUM_PERF_STATES * 100), (0x50, 100), + (0x60, 4), (0x64, 0x3f80_0000), (0x6c, 1), (0x70, 1), + (0x80, J714_GPU_PERF_BASE_PSTATE * 100), (0x84, 1), + (0x88, J714_GPU_NUM_PERF_STATES * 100), (0x90, 100), + + (0x9c, 1), (0xa0, 1), (0xa4, 1), (0xa8, 1), + ] { + put_u32(policy, offset, value); + } + + for index in 0..J714_POWER_POLICY_MINIMUM_ENTRIES { + put_u16(policy, J714_POWER_POLICY_MINIMUM_TABLE + index * 2, i16::MAX as u16); + } + + for (offset, value) in [ + (0xb58, 1), + (0xb60, 0x3f4c_cccd), + (0xb64, 0x3f7e_b852), + (0xb68, 0x3e4c_cccd), + (0xb6c, 0x3ba3_d70a), + (0xb70, 0x3f85_0e56), + (0xb74, 0x3f85_0e56), + (0xb80, 0x4060_0000), + (0xb84, 0x4060_0000), + ] { + put_u32(policy, offset, value); + } + put_u64(policy, M5_POLICY_STATE_PTR, bindings.state); + put_u64(policy, M5_POLICY_AON_PTD_PTR, bindings.aon_ptd); + Ok(()) +} + +/// Initial controller inputs for J714 with the matched calibration. +pub(crate) fn encode_m5_policy_controller_defaults(out: &mut [u8]) -> Result<(), InitdataError> { + if out.len() < M5_SHARED_PREFIX_SIZE { return Err(InitdataError::BufferSize); } + // Both calibrated hardware curves, as J714's firmware expects them. + const RELATIVE_POWER_MAIN: [u32; 16] = [ + 0x00000000, 0x4106f651, 0x413ea3b9, 0x4180c1c6, + 0x41b433fc, 0x41ccaf2d, 0x41f7a8ee, 0x42104b59, + 0x4228d2f7, 0x424956c5, 0x426d29a9, 0x428da278, + 0x42b4e595, 0x42c80000, 0x00000000, 0x00000000, + ]; + for (index, &value) in RELATIVE_POWER_MAIN.iter().enumerate() { put_u32(out, 0x19e4 + index * 4, value); } + const RELATIVE_POWER_AUX: [u32; 16] = [ + 0x00000000, 0x4105a0ed, 0x414a36f5, 0x418b74a8, + 0x41c1510d, 0x41df97fb, 0x420be387, 0x4223193c, + 0x423cc4e6, 0x42604354, 0x427ef922, 0x42957243, + 0x42b6ec22, 0x42c80000, 0x00000000, 0x00000000, + ]; + for (index, &value) in RELATIVE_POWER_AUX.iter().enumerate() { put_u32(out, 0x1a64 + index * 4, value); } + let policy = &mut out[M5_SHARED_LAYOUT.policy..M5_SHARED_LAYOUT.policy + M5_POLICY_SIZE]; + const INITIAL: &[(usize, u32)] = &[ + (0xd4, 0x3f828f5c), + (0xd8, 0x3f828f5c), + (0xdc, 0x3f828f5c), + (0xe0, 0x3f828f5c), + (0xe4, 0x3f828f5c), + (0xe8, 0x3f828f5c), + (0xec, 0x3f828f5c), + (0xf0, 0x3f828f5c), + (0xf4, 0x3f828f5c), + (0xf8, 0x3f828f5c), + (0xfc, 0x3f828f5c), + (0x100, 0x3f828f5c), + (0x104, 0x3f828f5c), + (0x108, 0x3f828f5c), + (0x9cc, 0x271), + (0x9d8, 0x3f7f2e9f), + (0x9e0, 0x3b516154), + (0x9e8, 0x3ca59586), + (0x9f4, 0x47c12200), + (0x9f8, 0x40a90fdb), + (0xa00, 0xbc576554), + (0xa04, 0x28), + (0xa08, J714_GPU_NUM_PERF_STATES * 100), + (0xa0c, J714_GPU_NUM_PERF_STATES * 100), + (M5_POLICY_MAXIMUM_POWER, 0x18244), + (0xa4c, J714_GPU_NUM_PERF_STATES * 100), + (0xa84, 0x8), + (0xa90, 0x3f000000), + (0xa98, 0x3f000000), + (0xaa0, 0x3e0b4396), + (0xaac, 0x47800000), + (0xab0, 0x3e99999a), + (0xabc, 0x28), + (0xac0, J714_GPU_NUM_PERF_STATES * 100), + (0xad0, 0x18244), + (0xadc, 0x10), + (0xae4, 0x5dc00), + (0xb3c, 0x4b), + (0xb44, 0x55), + (0xb48, 0x32), + (0xb4c, 0x6), + (0xb50, 0x1), + (0xb54, 0x6), + (0xb7c, 0x42be0000), + (0xb88, 0x414a1af3), + (0xb8c, 0x64), + (0xb90, J714_GPU_NUM_PERF_STATES * 100), + (0xb94, 0x64), + (0xba0, 0x4a), + (0xba8, 0x1), + (0xbd4, 0x64), + (0xbf4, 0x401), + (0xbf8, 0x6400), + (0xbfc, 0xa00), + (0xc00, 0x1f4000), + (0xc04, 0x12c00), + (0xc08, 0xfa000), + (0xc0c, 0x12c00), + (0xc10, 0x7d000), + (0xc14, 0x12c00), + (0xc18, 0x3e800), + (0xc1c, 0x19000), + (0xc20, 0x100), + (0xc78, 0x1), + (0x22ac, 0x47800000), + (0x22b0, 0x42c80000), + (0x22b4, 0x44a28000), + (0x22b8, 0x654), + (0x22cc, 0x64), + (0x22d4, 0x42c80000), + (0x22d8, J714_GPU_NUM_PERF_STATES * 100), + (0x22e0, 0x3f4ccccd), + (0x22e4, 0x3e4ccccd), + (0x2328, 0x6fcc3fcc), + (0x232c, 0x6fcc3fcc), + (0x234c, 0x1f40), + (0x2354, 0x4), + (0x2368, 0x3f800000), + (0x2370, 0x40ad5555), + (0x237c, 0x47800000), + (0x2380, 0x42080000), + (0x238c, 0x28), + (0x2390, J714_GPU_NUM_PERF_STATES * 100), + (0x2394, J714_GPU_NUM_PERF_STATES * 100), + (0x23a0, 0x2af8), + (0x23d4, J714_GPU_NUM_PERF_STATES * 100), + (0x2cec, 0x20), + (0x2cf0, 0x42000006), + (0x2cfc, 0x3), + (0x2d24, 0x1f40), + (0x2d2c, 0x4), + (0x2d40, 0x3f800000), + (0x2d48, 0x40855555), + (0x2d54, 0x47800000), + (0x2d58, 0x42080000), + (0x2d64, 0x64), + (0x2d68, J714_GPU_NUM_PERF_STATES * 100), + (0x2d6c, J714_GPU_NUM_PERF_STATES * 100), + (0x2d78, 0x2bc0), + (0x2dac, J714_GPU_NUM_PERF_STATES * 100), + (0x36c4, 0x20), + (0x36c8, 0x42000006), + (0x3fdc, 0xb8520000), + (0x3fe0, 0x3f7e), + (0x3fe4, 0xd70a0000), + (0x3fe8, 0x3ba3), + (0x3ff0, 0x43c8), + (0x3ffc, 0x4780), + (0x4000, 0x42c8), + (0x4008, 0x384bac8), + (0x400c, 0x5140000), + (0x4010, 0x3e80000), + (0x4018, 0x80000000), + (0x401c, 0x26ac453b), + (0x4050, 0x5140000), + (0x406c, 0xc80000), + (0x4124, J714_GPU_NUM_PERF_STATES * 100), + (0x4138, 0x1), + (0x41cc, 0x10001), + (0x41d0, 0x40000), + (0x41e8, 0x23f0000), + (0x41ec, 0x10000), + (0x41f0, 0x10000), + (0x4208, 0x2400000), + (0x420c, 0x10000), + (0x4210, 0x10000), + (0x4228, 0x2410000), + (0x42d4, 0x640000), + (0x42f8, 0x1), + (0x438c, 0x10001), + (0x4390, 0x40000), + (0x43a8, 0x25b0000), + (0x43ac, 0x10000), + (0x43b0, 0x10000), + (0x43c8, 0x25c0000), + (0x43cc, 0x10000), + (0x43d0, 0x10000), + (0x43e8, 0x25d0000), + (0x4494, 0x640000), + (0x449c, 0x1), + (0x44a4, 0x18244), + (0x44a8, 0x18244), + (0x44ac, 0x18244), + (0x44bc, 0x3f800000), + (0x44c0, 0x4), + (0x44c4, 0x8), + (0x44c8, 0x2ee00), + (0x4598, 0x7c), + (0x45a4, 0x3f77bdef), + (0x45ac, 0x3d042108), + (0x45b4, 0x3cf5c290), + (0x45c0, 0x47800000), + (0x45c4, 0x3f19999a), + (0x45d0, 0x28), + (0x45d4, J714_GPU_NUM_PERF_STATES * 100), + (0x45d8, J714_GPU_NUM_PERF_STATES * 100), + (0x45e0, 0x47c12200), + (0x45e4, 0x18244), + (0x45f0, 0xf8), + (0x45f8, 0x5ad200), + (0x4618, J714_GPU_NUM_PERF_STATES * 100), + (0x4658, 0x1), + (0x4728, 0x32), + (0x472c, 0x1), + (0x4734, 0x3f638e39), + (0x4738, 0x3f2aaaab), + (0x473c, 0x3de38e39), + (0x4740, 0x3eaaaaab), + (0x4744, 0xbecccccd), + (0x4748, 0xbecccccd), + (0x4750, 0x47800000), + (0x4754, 0xc0a00000), + (0x4758, 0xc0a00000), + (0x4760, 0x64), + (0x4764, J714_GPU_NUM_PERF_STATES * 100), + (0x4768, 0x64), + (0x4770, 0x45fa0000), + (0x4774, 0x578), + (0x477c, 0xe), + (0x4780, 0x48), + (0x4784, 0x18), + (0x4788, 0x1a5e00), + (0x4790, 0x8ca00), + (0x479c, 0x47800000), + (0x47c4, 0x1f40), + (0x47c8, 0x4), + (0x47cc, 0x9c4), + (0x47d0, 0x258), + (0x47d4, 0x2), + (0x47d8, 0x4), + (0x47dc, 0x32), + (0x480c, 0x47800000), + (0x481c, 0x28), + (0x4820, J714_GPU_NUM_PERF_STATES * 100), + (0x48ec, 0x1), + (0x48f0, 0x1), + (0x4910, 0xdac), + (0x4914, 0x3e8), + (0x4918, 0xbb8), + (0x49b8, J714_GPU_NUM_PERF_STATES * 100), + (0x49bc, J714_GPU_NUM_PERF_STATES * 100), + (0x49c0, 0x64), + (0x49c4, 0x20), + (0x49cc, 0x1), + (0x49d0, 0x1), + (0x49fc, 0x1f4), + (0x4a98, J714_GPU_NUM_PERF_STATES * 100), + (0x4a9c, J714_GPU_NUM_PERF_STATES * 100), + (0x4aa0, 0x64), + (0x4aa4, 0x20), + (0x4ac8, 0x4), + (0x4ad4, 0x3f780000), + (0x4adc, 0x3d000000), + (0x4ae4, 0x40a00000), + (0x4af0, 0x47800000), + (0x4af4, 0x41700000), + (0x4b00, 0x4b0), + (0x4b04, J714_GPU_NUM_PERF_STATES * 100), + (0x4b08, J714_GPU_NUM_PERF_STATES * 100), + (0x4b14, J714_GPU_NUM_PERF_STATES * 100), + (0x4b48, J714_GPU_NUM_PERF_STATES * 100), + (0x4b94, 0x42c80000), + (0x4ba4, 0x3), + (0x4ba8, 0x6fcc3fcc), + (0x4bac, 0x6fcc3fcc), + (0x4bcc, 0x7d), + (0x4bd0, 0x1), + (0x11a24, 0x67305452), + (0x11a28, 0x67315452), + (0x11a6c, 0x72305452), + (0x121a4, 0x1), + (0x122e0, 0x186a), + (0x121b0, 0x44d5c000), + (0x121b4, 0x44d20000), + (0x12230, 0x43140000), + (0x12234, 0x43140000), + (0x12280, 0x43d60000), + ]; + for &(offset, value) in INITIAL { put_u32(policy, offset, value); } + Ok(()) +} + +/// Initial controller inputs for J716c with its matched calibration: the J714 +/// defaults, including the relative power curves, with this chip's leakage +/// values (clusters 0-3, base, AFR) and its maximum power (196098 mW), and the +/// found masks of its AON PTD records (RT0g and RT1g, then RT0r) next to their +/// tags. +pub(crate) fn encode_m5_policy_controller_defaults_j716c(out: &mut [u8]) -> Result<(), InitdataError> { + encode_m5_policy_controller_defaults(out)?; + let policy = &mut out[M5_SHARED_LAYOUT.policy..M5_SHARED_LAYOUT.policy + M5_POLICY_SIZE]; + const MAXIMUM_POWER: u32 = 196_098; + const MAXIMUM_POWER_F32: u32 = 0x483f_8080; + for offset in [M5_POLICY_MAXIMUM_POWER, 0xad0, 0x44a4, 0x44a8, 0x44ac, 0x45e4] { + put_u32(policy, offset, MAXIMUM_POWER); + } + for offset in [0x9f4, 0x45e0] { + put_u32(policy, offset, MAXIMUM_POWER_F32); + } + for (index, value) in [0x4524_9000, 0x451f_9000, 0x451f_9000, 0x4524_9000].into_iter().enumerate() { + put_u32(policy, 0x121b0 + index * 4, value); + } + for index in 0..4 { + put_u32(policy, 0x12230 + index * 4, 0x434d_8000); + } + put_u32(policy, 0x12280, 0x44b1_8000); + put_u32(policy, 0x11a64, 3); + put_u32(policy, 0x11a74, 1); + Ok(()) +} + +/// Controller inputs for J775D: the J714 defaults, with the Ultra's power model over them. The +/// model covers the main and auxiliary operating points of both dies (64 cores, 110 C) with +/// synthetic leakage inputs (1350 per cluster core and per die auxiliary domain, 51 per cluster +/// SRAM), which hold for every J775D: P13 sums to 345262 mW. Then the relative power curves of +/// that model, the controllers' gains and utilization thresholds, and the temperature-sensor +/// channel masks of both dies (160 GPU and 8 auxiliary-domain channels; the hotspot set is both). +pub(crate) fn encode_m5_policy_controller_defaults_j775d(out: &mut [u8]) -> Result<(), InitdataError> { + encode_m5_policy_controller_defaults(out)?; + const RELATIVE_POWER_MAIN: [u32; 14] = [ + 0x0000_0000, 0x4107_2435, 0x414b_7ecd, 0x4187_dfa6, 0x41be_759c, 0x41d6_f4a2, 0x41ff_340b, + 0x4215_4139, 0x4226_cc22, 0x4247_2db8, 0x426b_3fa4, 0x428b_aed3, 0x42b6_c65b, 0x42c8_0000, + ]; + for (index, &value) in RELATIVE_POWER_MAIN.iter().enumerate() { put_u32(out, 0x19e4 + index * 4, value); } + const RELATIVE_POWER_AUX: [u32; 14] = [ + 0x0000_0000, 0x4106_2d7e, 0x4154_96b9, 0x4189_3d11, 0x41ba_e22e, 0x41d5_33ed, 0x4202_7bcd, + 0x421c_12dd, 0x4230_6915, 0x4250_6a36, 0x427d_e40c, 0x4294_0cc0, 0x42bd_bd8e, 0x42c8_0000, + ]; + for (index, &value) in RELATIVE_POWER_AUX.iter().enumerate() { put_u32(out, 0x1a64 + index * 4, value); } + let policy = &mut out[M5_SHARED_LAYOUT.policy..M5_SHARED_LAYOUT.policy + M5_POLICY_SIZE]; + const MAXIMUM_POWER: u32 = 345_262; + const MAXIMUM_POWER_F32: u32 = 0x48a8_95c0; + for offset in [M5_POLICY_MAXIMUM_POWER, 0xad0, 0x44a4, 0x44a8, 0x44ac, 0x45e4] { + put_u32(policy, offset, MAXIMUM_POWER); + } + for offset in [0x9f4, 0x45e0] { + put_u32(policy, offset, MAXIMUM_POWER_F32); + } + // Leakage: eight clusters' cores (1350.0) and SRAM (51.0), then each die's auxiliary domain. + for index in 0..8 { + put_u32(policy, 0x121b0 + index * 4, 0x44a8_c000); + put_u32(policy, 0x12230 + index * 4, 0x424c_0000); + } + for index in 0..2 { + put_u32(policy, 0x12280 + index * 4, 0x44a8_c000); + } + // The power controller's integral and proportional gains (0.08, 0.005), the second + // controller's (0.01, 0.28), and the utilization thresholds 75 and 95. + for (offset, value) in [(0xaa0, 0x3da3_d70b), (0xab0, 0x3ba3_d70a), (0x45b4, 0x3c23_d70b), + (0x45c4, 0x3e8f_5c29), (0xb44, 75), (0xb48, 95)] { + put_u32(policy, offset, value); + } + // The performance controllers' floor, P3, as state x 100 in the three floor words (the last + // byte-packed in the boost block); HwData's relative-frequency tables are based at it too. + for offset in [0xb94, 0xbd4, 0xbf9] { + put_u32(policy, offset, J775D_PERF_FLOOR as u32 * 100); + } + // The fast-active boost may continue while the GPU idles for 20 percent of a boost window + // (byte-packed in the boost block) on the Ultra's core type; the default is 10. + policy[0xbfd] = 20; + // The AON temperature records the GPU's and the auxiliary domain's controllers read, with + // each list's found mask: both dies' g0TR-g3TR and r0TR-r1TR, the six records of J775D's + // AON table (encode_common26_aon_ptd). + for (index, tag) in [0x6730_5452, 0x6731_5452, 0x6732_5452, 0x6733_5452].into_iter().enumerate() { + put_u32(policy, 0x11a24 + index * 4, tag); + } + put_u32(policy, 0x11a64, 0xf); + for (index, tag) in [0x7230_5452, 0x7231_5452].into_iter().enumerate() { + put_u32(policy, 0x11a6c + index * 4, tag); + } + put_u32(policy, 0x11a74, 0x3); + const DIE_CHANNELS: u64 = 0x6fcc_3fcc_6fcc_3fcc; + for (offset, words) in [ + (0x2324, [0x6fcc_3fcc_0000_0000, DIE_CHANNELS, DIE_CHANNELS, DIE_CHANNELS, 0x6fcc_3fcc]), + (0x2cfc, [0x0303_0303, 0, 0, 0, 0]), + (0x4ba4, [0x6fcc_3fcc_0303_0303, DIE_CHANNELS, DIE_CHANNELS, DIE_CHANNELS, 0x6fcc_3fcc]), + ] { + for (index, word) in words.into_iter().enumerate() { + put_u64(policy, offset + index * 8, word); + } + } + Ok(()) +} diff --git a/drivers/gpu/drm/asahi/g17/fw/m5/mod.rs b/drivers/gpu/drm/asahi/g17/fw/m5/mod.rs new file mode 100644 index 0000000000000..5c3eef8c54e22 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fw/m5/mod.rs @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! M5 firmware-specific work encodings, using the shared G17 queue primitives. + +pub(crate) mod initdata; +pub(crate) mod power; + +/// M5 adds a statistics record at tag 32, beyond the legacy 32-bit mask. +pub(crate) const STATISTICS_TYPES: u64 = 0x1_5fff_0000; + +/// M5 stores an entry count in the trailer, with its upper half clear. +/// The preceding register storage and self pointer use the shared layout. +pub(crate) fn set_register_array( + array: &mut super::kick::RegisterArray, + address: u64, + writes: &[super::kick::RegisterWrite], +) -> kernel::prelude::Result { + array.set(address, writes)?; + array.size = writes.len() as u32; + Ok(()) +} + +pub(crate) mod pool; +pub(crate) mod pool_budget; +pub(crate) mod queue; +pub(crate) mod compute; + +pub(crate) mod render_demand; + +pub(crate) mod render; +pub(crate) mod render_tail; + +pub(crate) mod render_registers; + +pub(crate) mod render_assembly; + +pub(crate) mod target; diff --git a/drivers/gpu/drm/asahi/g17/fw/m5/pool.rs b/drivers/gpu/drm/asahi/g17/fw/m5/pool.rs new file mode 100644 index 0000000000000..b8c52bc634c5f --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fw/m5/pool.rs @@ -0,0 +1,283 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! M5 private-memory allocation encodings. +#![allow(dead_code)] + +use core::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; + +use super::initdata::{Board, J714}; + +pub(crate) const BLOCK_BYTES: usize = 0x20_0000; +// Render block size and private-memory address inventory of the M5 +// part, selected at probe (`Board::render_pool`). The inventory sizes the +// directory, not the physical allocation; each pool allocates only its +// requested initial blocks. +static RENDER_BLOCK_BYTES: AtomicUsize = AtomicUsize::new(J714.render_pool.0); +static POOL_BYTES: AtomicU64 = AtomicU64::new(J714.render_pool.1); + +/// Selects the board's render pool before any pool is built. +pub(crate) fn select_render_pool(board: &Board) { + let (block, pool) = board.render_pool; + RENDER_BLOCK_BYTES.store(block, Ordering::Relaxed); + POOL_BYTES.store(pool, Ordering::Relaxed); +} + +pub(crate) fn render_block_bytes() -> usize { + RENDER_BLOCK_BYTES.load(Ordering::Relaxed) +} + +const RING_MAX_PAGES: u64 = 0x3ff800; +pub(crate) const OPERAND_BYTES: usize = 0x2_0000; +pub(crate) const STATE_BYTES: usize = 0x100; +/// Unaligned firmware-completed submission counter. +pub(crate) const COMPLETED: usize = 0x74; +/// Mutable words saved from the descriptor after the pool's final release. +pub(crate) const SAVED_FIELDS: [usize; 8] = [0x28, 0x20, 0x24, 0x2c, 0x3c, 0x48, 0x38, 0x4c]; +const GPU_PAGE: usize = 0x4000; +const PRIVATE_TOP: u64 = 1 << 42; +pub(crate) const SCENE_TABLE_ENTRIES: usize = 0x50; +pub(crate) const SCENE_STRIDE: usize = 0x80; +pub(crate) const SCENE_TRAILER_SIZE: usize = 0x40; + +pub(crate) fn scene_table_end(address: u64) -> Option { + address.checked_add((SCENE_TABLE_ENTRIES * SCENE_STRIDE) as u64) +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum PoolError { Geometry, Address, Buffer } + +/// Snapshot of the retained pool header. Address and allocation geometry are +/// host-owned; only SAVED_FIELDS may be restored from a retired descriptor. +pub(crate) struct PoolState(pub(crate) [u8; STATE_BYTES]); + +impl PoolState { + fn u32(&self, offset: usize) -> u64 { + u64::from(u32::from_le_bytes(self.0[offset..offset + 4].try_into().unwrap())) + } + fn u64(&self, offset: usize) -> u64 { + u64::from_le_bytes(self.0[offset..offset + 8].try_into().unwrap()) + } + pub(crate) fn descriptor(&self) -> [u64; 8] { + [self.u64(0x14) >> 7 | self.u32(0x28) << 61, + self.u64(0x20), self.u32(0x2c) | self.u32(0x1c) << 32, 0, + self.u64(0x30) >> 6 | self.u32(0x40) << 44, + self.u32(0x3c) | self.u32(0x44) << 31, + self.u32(0x48), self.u32(0x38) | self.u32(0x4c) << 32] + } + pub(crate) fn restore(&mut self, words: [u64; 8]) { + let values = [words[0] >> 61 & 1, words[1] & 0x0fff_ffff, + words[1] >> 32 & 0x0fff_ffff, words[2] & 0x0fff_ffff, + words[5] & 0x0fff_ffff, words[6] & 0xffff, + words[7] & 0x0fff_ffff, words[7] >> 32 & 0xffff]; + for (offset, value) in SAVED_FIELDS.into_iter().zip(values) { + self.0[offset..offset + 4].copy_from_slice(&(value as u32).to_le_bytes()); + } + } + /// Reject a descriptor belonging to different backing before restoring + /// its mutable counters. Reserved bits are not host address bits. + pub(crate) fn matches(&self, words: &[u64; 8]) -> bool { + let own = self.descriptor(); + words[0] & ((1 << 35) - 1) == own[0] & ((1 << 35) - 1) + && words[2] >> 32 == own[2] >> 32 + && words[4] == own[4] + } +} + +/// Parameter-buffer capacity follows the memory budget and initial demand. +/// Capacity is rounded to groups of 33 half-MiB blocks; it reserves list space, +/// not physical backing for every possible block. +pub(crate) fn parameter_capacity(memory_bytes: u64, initial_bytes: u64) -> Result { + const GROUP: u64 = 33 * 0x80000; + let scaled = memory_bytes.checked_mul(5).ok_or(PoolError::Geometry)? >> 2; + let budget = (((u128::from(scaled) * 0x28f5_c28f_5c28_f5c3) >> 64) as u64) >> 2; + let groups = |bytes: u64| bytes.checked_add(GROUP - 1) + .map(|n| n / GROUP).ok_or(PoolError::Geometry); + let groups = groups(budget)?.max(groups(initial_bytes)?); + usize::try_from(groups.checked_mul(33).ok_or(PoolError::Geometry)?) + .map_err(|_| PoolError::Geometry) +} + +/// Per-scene scratch covers the complete parameter-buffer inventory. Optional +/// cluster entries are supplied by the dispatch policy, not the active draw's +/// physical block count. Each entry and the header occupy eight bytes. +pub(crate) fn parameter_scratch_stride(capacity: usize, cluster_entries: usize) + -> Result +{ + if capacity == 0 { return Err(PoolError::Geometry); } + let pages = capacity.checked_mul(16).ok_or(PoolError::Geometry)?; + let entries = pages.checked_add(0x17ff).ok_or(PoolError::Geometry)? / 0x1800; + let entries = entries.checked_add(cluster_entries).ok_or(PoolError::Geometry)?; + entries.checked_mul(8).and_then(|n| n.checked_add(8 + 63)) + .map(|n| n & !63).ok_or(PoolError::Geometry) +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct PoolGeometry { + pub(crate) blocks: usize, + pub(crate) directory_entries: u32, + pub(crate) pages: u32, + pub(crate) directory_bytes: usize, + pub(crate) freelist_bytes: usize, + g17s_render_blocks: bool, +} + +fn round_page(bytes: usize) -> Result { + bytes.checked_add(GPU_PAGE - 1).map(|n| n & !(GPU_PAGE - 1)).ok_or(PoolError::Geometry) +} + +impl PoolGeometry { + pub(crate) fn g17s(blocks: usize) -> Result { + let bytes = POOL_BYTES.load(Ordering::Relaxed).min(RING_MAX_PAGES << 12) + & !(render_block_bytes() as u64 - 1); + Self::g17s_render(blocks, (bytes >> 12) as u32) + } + pub(crate) fn new(blocks: usize, directory_entries: u32) -> Result { + Self::with_kind(blocks, directory_entries, false) + } + + pub(crate) fn g17s_render(blocks: usize, directory_entries: u32) -> Result { + Self::with_kind(blocks, directory_entries, true) + } + + pub(crate) fn block_bytes(self) -> usize { + if self.g17s_render_blocks { render_block_bytes() } else { BLOCK_BYTES } + } + + fn with_kind(blocks: usize, directory_entries: u32, g17s_render_blocks: bool) -> Result { + let block_bytes = if g17s_render_blocks { render_block_bytes() } else { BLOCK_BYTES }; + let pages = blocks.checked_mul(block_bytes / 0x1000) + .and_then(|n| u32::try_from(n).ok()).ok_or(PoolError::Geometry)?; + if blocks == 0 || pages > directory_entries { return Err(PoolError::Geometry); } + let directory_bytes = round_page((directory_entries as usize).checked_mul(8) + .ok_or(PoolError::Geometry)?)?; + let stride = if g17s_render_blocks { 8 } else { 0x40 }; + let freelist_bytes = round_page(blocks.checked_mul(stride).ok_or(PoolError::Geometry)?)?; + + if freelist_bytes >> 12 >= 1 << 20 { return Err(PoolError::Geometry); } + Ok(Self { blocks, directory_entries, pages, directory_bytes, freelist_bytes, g17s_render_blocks }) + } +} + +fn private_range(address: u64, bytes: usize) -> Result<(), PoolError> { + if address == 0 || address & (GPU_PAGE as u64 - 1) != 0 + || address.checked_add(bytes as u64).is_none_or(|end| end > PRIVATE_TOP) { + Err(PoolError::Address) + } else { Ok(()) } +} + +fn validate_blocks(g: PoolGeometry, blocks: &[u64]) -> Result<(), PoolError> { + if PoolGeometry::with_kind(g.blocks, g.directory_entries, g.g17s_render_blocks)? != g { return Err(PoolError::Geometry); } + if blocks.len() != g.blocks { return Err(PoolError::Geometry); } + for address in blocks { private_range(*address, g.block_bytes())?; } + Ok(()) +} + +pub(crate) fn encode_directory(g: PoolGeometry, blocks: &[u64], out: &mut [u8]) + -> Result<(), PoolError> +{ + validate_blocks(g, blocks)?; + if out.len() < g.directory_bytes { return Err(PoolError::Buffer); } + out[..g.directory_bytes].fill(0); + + let pages_per_block = g.block_bytes() / 0x1000; + for (block, address) in blocks.iter().enumerate() { + for page in 0..pages_per_block { + let offset = (block * pages_per_block + page) * 8; + out[offset..offset + 8].copy_from_slice(&(address + page as u64 * 0x1000).to_le_bytes()); + } + } + Ok(()) +} + +pub(crate) fn encode_freelist(g: PoolGeometry, blocks: &[u64], out: &mut [u8]) + -> Result<(), PoolError> +{ + validate_blocks(g, blocks)?; + if out.len() < g.freelist_bytes { return Err(PoolError::Buffer); } + out[..g.freelist_bytes].fill(0); + for (index, address) in blocks.iter().enumerate() { + let (offset, value) = if g.g17s_render_blocks { (index * 8, *address) } + else { (index * 0x40, address | (1 << 61)) }; + out[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); + } + Ok(()) +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct PoolLayout { + pub(crate) geometry: PoolGeometry, + pub(crate) directory: u64, + pub(crate) freelist: u64, + + pub(crate) hardware_id: u32, +} + +fn validate_layout(p: PoolLayout) -> Result<(), PoolError> { + if PoolGeometry::with_kind(p.geometry.blocks, p.geometry.directory_entries, p.geometry.g17s_render_blocks)? != p.geometry { + return Err(PoolError::Geometry); + } + private_range(p.directory, p.geometry.directory_bytes)?; + private_range(p.freelist, p.geometry.freelist_bytes) +} + +pub(crate) fn initial_uma_descriptor(p: PoolLayout) -> Result<[u64; 8], PoolError> { + validate_layout(p)?; + if p.geometry.g17s_render_blocks { + let blocks = p.geometry.blocks as u64; + return Ok([p.directory >> 7, u64::from(p.geometry.pages) << 32, + u64::from(p.geometry.pages) | (u64::from(p.geometry.directory_entries) << 32), + 0, (p.freelist >> 6) | ((p.geometry.block_bytes() as u64 >> 12) << 44), + blocks, 0, blocks]); + } + Ok([p.directory >> 7, u64::from(p.geometry.pages) << 32, + u64::from(p.geometry.pages) | (u64::from(p.geometry.directory_entries) << 32), + 0, (p.freelist >> 6) | ((p.geometry.freelist_bytes as u64 >> 12) << 44), + 0, 0, 0]) +} + +pub(crate) fn encode_state(p: PoolLayout, submitted: u64, out: &mut [u8]) + -> Result<(), PoolError> +{ + validate_layout(p)?; + if submitted == 0 || submitted & 3 != 0 { return Err(PoolError::Address); } + if out.len() < STATE_BYTES { return Err(PoolError::Buffer); } + out[..STATE_BYTES].fill(0); + for (offset, value) in [(0, 1u32), (8, p.hardware_id), (0x10, 2), + (0x1c, p.geometry.directory_entries), (0x24, p.geometry.pages), + (0x2c, p.geometry.pages), (0x40, (p.geometry.freelist_bytes >> 12) as u32), + (0x60, 2), (0x80, 1)] { + out[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); + } + for (offset, value) in [(0x14, p.directory), (0x30, p.freelist), + (0x50, p.freelist), (0x6c, submitted)] { + out[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); + } + if p.geometry.g17s_render_blocks { + + for (offset, value) in [(0x0c, 2), (0x38, p.geometry.blocks as u32), + (0x3c, p.geometry.blocks as u32), + (0x40, (p.geometry.block_bytes() >> 12) as u32), (0x44, 0), + (0x60, 0), + (0x68, p.geometry.blocks as u32), (0x80, 4)] { + out[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); + } + } + Ok(()) +} + +pub(crate) fn encode_g17s_render_state(p: PoolLayout, submitted: u64, out: &mut [u8]) + -> Result<(), PoolError> +{ + encode_state(p, submitted, out)?; + out[0x80..0x84].copy_from_slice(&4u32.to_le_bytes()); + Ok(()) +} + +pub(crate) fn encode_g17s_compute_state(p: PoolLayout, submitted: u64, out: &mut [u8]) + -> Result<(), PoolError> +{ + if !p.geometry.g17s_render_blocks { return Err(PoolError::Geometry); } + encode_state(p, submitted, out)?; + out[0x7c..0x80].copy_from_slice(&1u32.to_le_bytes()); + out[0x80..0x84].copy_from_slice(&1u32.to_le_bytes()); + Ok(()) +} diff --git a/drivers/gpu/drm/asahi/g17/fw/m5/pool_budget.rs b/drivers/gpu/drm/asahi/g17/fw/m5/pool_budget.rs new file mode 100644 index 0000000000000..ce0b5dc6129a4 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fw/m5/pool_budget.rs @@ -0,0 +1,153 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! G17S shared render private-memory demand. Counts describe hardware and +//! simultaneous work; byte requirements come from the submitted programs. + +use super::pool::{render_block_bytes, PoolError, PoolGeometry}; + +#[derive(Clone, Copy)] +pub(crate) struct RenderBudget { + active_cores: u32, + constant_pages: u32, + active_kicks: u32, +} + +#[derive(Clone, Copy, Default)] +pub(crate) struct Requirements { + pub(crate) minimum_bytes: u64, + pub(crate) ideal_bytes: u64, + pub(crate) self_reserve_pages: u32, + pub(crate) render_target_pages: u32, + pub(crate) has_spill: bool, +} + +/// Retained high-water marks, excluding per-kick reservations from the two +/// base byte counts. They may increase demand but never replace live demand. +#[derive(Clone, Copy, Default)] +pub(crate) struct History { + pub(crate) minimum_base: u64, + pub(crate) ideal_base: u64, + pub(crate) self_reserve: u64, + pub(crate) render_target: u64, + pub(crate) minimum_pool: u64, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct Sizes { + pub(crate) minimum: u64, + pub(crate) ideal: u64, +} + +fn add(a: u64, b: u64) -> Result { + a.checked_add(b).ok_or(PoolError::Geometry) +} +fn mul(a: u64, b: u64) -> Result { + a.checked_mul(b).ok_or(PoolError::Geometry) +} +fn aligned(bytes: u64) -> Result { + Ok(add(bytes, render_block_bytes() as u64 - 1)? + & !(render_block_bytes() as u64 - 1)) +} + +impl RenderBudget { + /// `core_masks` contains the enabled cores of each physical cluster. + /// `ptc_floor` is the configured per-core context count; `active_kicks` + /// includes simultaneously resident tiling and fragment work. + pub(crate) fn new(core_masks: &[u32], ptc_floor: u32, active_kicks: u32) + -> Result + { + let mut active_cores = 0u32; + let mut cluster_cores = 0u32; + for mask in core_masks { + let count = mask.count_ones(); + active_cores = active_cores.checked_add(count).ok_or(PoolError::Geometry)?; + cluster_cores = cluster_cores.max(count); + } + if active_cores == 0 || active_kicks == 0 || ptc_floor == 0 { + return Err(PoolError::Geometry); + } + let constant_pages = cluster_cores.checked_mul(ptc_floor) + .and_then(|n| n.checked_add(32)) + .and_then(|n| n.checked_mul(u32::try_from(core_masks.len()).ok()?)) + .ok_or(PoolError::Geometry)?; + Ok(Self { active_cores, constant_pages, active_kicks }) + } + + /// Compute SCS storage is private to each enabled USC. The auxiliary + /// allocation the dispatch publishes, per USC and at least the eight-page + /// tail, belongs to the compute allocator, in addition to its small/large + /// SCS page inventories and the execution-state spill reservation. As for + /// render, the pool covers what the descriptor tells the hardware. + pub(crate) fn compute_scs_requirements(&self, mut request: Requirements, + pages: u16, auxiliary_pages: u32) -> Result + { + if pages != 0 { + let small = u64::from(pages); + let per_core = add(add(small, small.div_ceil(256))?, u64::from(auxiliary_pages.max(8)))?; + let bytes = mul(mul(per_core, u64::from(self.active_cores))?, 4096)?; + request.minimum_bytes = add(request.minimum_bytes, bytes)?; + request.ideal_bytes = add(request.ideal_bytes, bytes)?; + request.has_spill = true; + } + Ok(request) + } + + /// Render's allocator publishes the already rounded auxiliary page count + /// in the descriptor, including its eight-page tail. This is separate + /// from the layout-derived small/large SCS inventories. The USC + /// spill-buffer allocation multiplies their sum by cores. + pub(crate) fn render_scs_requirements(&self, mut request: Requirements, + pages: u16, auxiliary_pages: u32) -> Result + { + if pages != 0 { + let small = u64::from(pages); + let per_core = add(add(small, small.div_ceil(256))?, + u64::from(auxiliary_pages))?; + let bytes = mul(mul(per_core, u64::from(self.active_cores))?, 4096)?; + request.minimum_bytes = add(request.minimum_bytes, bytes)?; + request.ideal_bytes = add(request.ideal_bytes, bytes)?; + request.has_spill = true; + } + Ok(request) + } + + pub(crate) fn sizes(&self, request: Requirements, history: History) + -> Result + { + let reserve = u64::from(request.self_reserve_pages) << 12; + let rt = u64::from(request.render_target_pages) << 12; + let reserved = add(reserve, rt)?; + if request.minimum_bytes < reserved || request.ideal_bytes < reserved { + return Err(PoolError::Geometry); + } + // Every enabled USC has execution-state storage. Programs with spill + // state add twelve pages; shared queues reserve another twenty-four. + let execution = mul(u64::from(self.active_cores), + if request.has_spill { 76 << 12 } else { 64 << 12 })?; + let constant = mul(u64::from(self.constant_pages), + u64::from(self.active_kicks - 1) << 12)?; + let per_kick = mul(add(reserve.max(history.self_reserve), + rt.max(history.render_target))?, u64::from(self.active_kicks))?; + let size = |base: u64, high_water: u64| { + let base = add(base, execution)?.checked_sub(reserved) + .ok_or(PoolError::Geometry)?.max(high_water); + add(add(base, constant)?, per_kick) + }; + let ideal = aligned(size(request.ideal_bytes, history.ideal_base)?)?; + let minimum = aligned(size(request.minimum_bytes, history.minimum_base)? + .min(ideal.max(history.minimum_pool)))?; + if minimum > ideal { return Err(PoolError::Geometry); } + Ok(Sizes { minimum, ideal }) + } +} + +impl Sizes { + /// Reserve the full address inventory but allocate only the desired + /// physical blocks. The hardware directory limit is checked by geometry. + pub(crate) fn geometry(self) -> Result { + if self.minimum == 0 || self.minimum > self.ideal { return Err(PoolError::Geometry); } + let blocks = usize::try_from(self.ideal / render_block_bytes() as u64) + .map_err(|_| PoolError::Geometry)?; + if self.ideal % render_block_bytes() as u64 != 0 { return Err(PoolError::Geometry); } + PoolGeometry::g17s(blocks) + } +} diff --git a/drivers/gpu/drm/asahi/g17/fw/m5/power.rs b/drivers/gpu/drm/asahi/g17/fw/m5/power.rs new file mode 100644 index 0000000000000..9d0ab94f392e8 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fw/m5/power.rs @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Live M5 Region C inputs. These offsets do not follow the Neo layout. + +pub(crate) const EFFORT: usize = 0xe8; +pub(crate) const EFFORT_OVERRIDE: usize = 0xec; +pub(crate) const POWER_ASSERT_TALLY: usize = 0x1c99; + +/// Control records which assert power among the opcodes this driver emits. +pub(crate) const fn asserts_gpu_power(opcode: u32) -> bool { + matches!(opcode, 0x08 | 0x0a | 0x29) +} diff --git a/drivers/gpu/drm/asahi/g17/fw/m5/queue.rs b/drivers/gpu/drm/asahi/g17/fw/m5/queue.rs new file mode 100644 index 0000000000000..0a0b64b1de05d --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fw/m5/queue.rs @@ -0,0 +1,202 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! M5 meanings of fields in the shared channel configuration record. +use kernel::prelude::*; +use super::super::queue::{Policy,QueueConfig,QueueConfigArgs,QueueConfigTarget}; + +/// The firmware's channel priority profiles, including the independent QoS +/// weight selected at queue registration. A single owner has a full share. +pub(crate) fn scheduling_policy(priority: u32) -> Result { + match priority { + 0 => Ok(Policy::new(0, 1, 15, 8192)), + 1 => Ok(Policy::new(1, 0, 20, 8192)), + 2 => Ok(Policy::new(2, 2, 4, 8192)), + 3 => Ok(Policy::new(3, 0, 20, 8192)), + _ => Err(EINVAL), + } +} + +/// M5's channel-state record. The scheduler pointer sits earlier than in +/// the Neo record; the policy and owner fields also moved. +#[repr(C, packed)] +#[derive(Clone, Copy)] +pub(crate) struct QueueRecord { + pointers_va: u64, + items_va: u64, + job_list_va: u64, + reserved_18: [u8; 0xc], + sentinel_24: u32, + priority: [u32; 2], + reserved_30: [u8; 6], + sentinel_36: u16, + reserved_38: u32, + policy: u32, + sentinel_40: u32, + owner_pid: u32, + reserved_48: [u8; 0x28], + scheduler_va: u64, + reserved_78: [u8; 8], +} +impl QueueRecord { + fn scheduling_words(policy: Policy) -> Result<([u8; 6], u16, u32)> { + match (policy.priority(), policy.policy()) { + (0, 1) => Ok(([0, 0, 0xff, 0xff, 0xff, 0xff], u16::MAX, 1)), + (1, 0) => Ok(([0, 0, 0, 0, 0xff, 0xff], u16::MAX, 0)), + (2, 2) => Ok(([0; 6], u16::MAX, 0)), + (3, 0) => Ok(([0; 6], 0, 0)), + _ => Err(EINVAL), + } + } + /// The scheduling words follow the queue's priority class. + pub(crate) fn new(args: &super::super::queue::QueueRecordArgs) -> Result { + let (reserved_30, sentinel, reserved_38) = Self::scheduling_words(args.policy)?; + Ok(Self { + pointers_va: args.pointers_va, items_va: args.items_va, + job_list_va: args.job_list_va, reserved_18: [0; 0xc], + sentinel_24: u32::MAX, priority: [u32::from(args.policy.priority()); 2], reserved_30, + sentinel_36: sentinel, reserved_38, policy: u32::from(args.policy.policy()), + sentinel_40: u32::MAX, owner_pid: args.owner_pid, + reserved_48: [0; 0x28], scheduler_va: args.scheduler_va, + reserved_78: [0; 8], + }) + } + /// Every host-owned identity word already holds what set_compute_owner + /// would store (so that update would not change the record). + pub(crate) fn names_owner(&self, work_head: u64, scheduler: u64, + owner_pid: u32, policy: super::super::queue::Policy) -> Result { + let (reserved_30, sentinel, reserved_38) = Self::scheduling_words(policy)?; + let (job_list, scheduler_va, pid, priority) = + (self.job_list_va, self.scheduler_va, self.owner_pid, self.priority); + let (r30, s36, r38, current) = (self.reserved_30, self.sentinel_36, self.reserved_38, self.policy); + Ok(job_list == work_head && scheduler_va == scheduler && pid == owner_pid + && priority == [u32::from(policy.priority()); 2] && r30 == reserved_30 + && s36 == sentinel && r38 == reserved_38 && current == u32::from(policy.policy())) + } + /// Replace only host-owned identity fields after the old owner retires. + pub(crate) fn set_compute_owner(&mut self, work_head: u64, scheduler: u64, + owner_pid: u32, policy: super::super::queue::Policy) -> Result { + let (reserved_30, sentinel, reserved_38) = Self::scheduling_words(policy)?; + // Changing a channel's priority invalidates this word when its hardware + // class changes. Retained physical queues can change class on reuse. + if self.priority[0] != u32::from(policy.priority()) { + self.sentinel_40 = u32::MAX; + } + self.job_list_va = work_head; + self.scheduler_va = scheduler; + self.owner_pid = owner_pid; + self.priority = [u32::from(policy.priority()); 2]; + self.reserved_30 = reserved_30; + self.sentinel_36 = sentinel; + self.reserved_38 = reserved_38; + self.policy = u32::from(policy.policy()); + Ok(()) + } +} +const _: () = { + assert!(core::mem::size_of::() == 0x80); + assert!(core::mem::offset_of!(QueueRecord, policy) == 0x3c); + assert!(core::mem::offset_of!(QueueRecord, scheduler_va) == 0x70); +}; + +pub(crate) struct ConfigProfile { + pub(crate) soft_fault_enabled:bool, + pub(crate) extended:bool, + pub(crate) parameter_manager:Option, +} +pub(crate) fn config(args:&QueueConfigArgs,p:ConfigProfile)->Result { + if p.parameter_manager.is_some() && !matches!(args.target,QueueConfigTarget::Tiling(Some(_))) { + return Err(EINVAL); + } + let mut out=QueueConfig::new(args)?; + out.unk_66=u16::from(p.soft_fault_enabled); + out.unk_68[2..6].copy_from_slice(&u32::from(p.extended).to_le_bytes()); + if let Some(manager)=p.parameter_manager {out.fragment_qid=manager;} + Ok(out) +} + +/// M5 publishes the current kick with the common tag-16 wire shape. All +/// reserved bytes are zero; the descriptor is reached through the shared ring's entry. +#[repr(C, packed)] +#[derive(Clone, Copy)] +pub(crate) struct Notification { + record: super::super::queue::KickPredecessor, + reserved: [u8; 6], +} +pub(crate) fn notification(qid:u8, current:super::super::kick::KickTimestamp)->Result { + Ok(Notification{record:super::super::queue::KickPredecessor::new(qid,current)?,reserved:[0;6]}) +} +const _: () = assert!(core::mem::size_of::() == 24); + +/// Host-port transaction for an M5 render pair. The FIFO is a register pair +/// with a 16-byte stride, selecting slot 9. Queue validity +/// is asserted high then low, after its configuration payload. An existing +/// configuration is invalidated in the same order before replacement. +#[derive(Clone, Copy)] +pub(crate) struct HardwareSubmission { + writes: [(usize,u64);16], + count: usize, +} +impl HardwareSubmission { + pub(crate) fn advance_pair(fragment:u8,tiling:u8)->Result { + if fragment==tiling {return Err(EINVAL);} + let mut writes=[(0,0);16]; + for (n,qid) in [fragment,tiling].into_iter().enumerate() { + writes[n*2..n*2+2].copy_from_slice(&[ + (0x4090,Self::configuration_tag(qid)?+0x10),(0x4098,1)]); + } + Ok(Self{writes,count:4}) + } + /// Configure the compute queue at first use, then publish only new kicks. + pub(crate) fn compute(qid:u8,va:u64,priority:u8,configure:bool)->Result { + use super::super::queue::{Policy,DataMaster}; + const ADDRESS_MASK:u64=0x0000_07ff_ffff_ffe0; + let tag=Self::configuration_tag(qid)?; + if priority>=Policy::PRIORITIES || va & !ADDRESS_MASK != 0 {return Err(EINVAL);} + let mut writes=[(0,0);16]; + let mut count=0; + if configure { + let (low,high)=if qid<64 {(1u64<Result { + use super::super::{kick::QID_MAX,queue::{Policy,DataMaster}}; + const ADDRESS_MASK:u64=0x0000_07ff_ffff_ffe0; + if fragment.0==tiling.0 || priority>=Policy::PRIORITIES {return Err(EINVAL);} + let mut writes=[(0,0);16]; + let mut count=0; + for ((qid,va),engine) in [(fragment,DataMaster::Fragment),(tiling,DataMaster::Tiling)] { + if qid>QID_MAX || va & !ADDRESS_MASK != 0 {return Err(EINVAL);} + let completion=(3*(4-u64::from(priority))+0x3fe).wrapping_shl(54); + let tag=0x0020_0800_0002_1008 | (u64::from(qid)<<44); + let payload=((engine as u64+1)<<60)|va|completion; + let (low,high)=if qid<64 {(1u64<Result { + if qid>super::super::kick::QID_MAX {return Err(EINVAL);} + Ok(0x0020_0800_0002_1008 | (u64::from(qid)<<44)) + } + pub(crate) fn writes(&self)->&[(usize,u64)] {&self.writes[..self.count]} +} + +/// M5's scheduler release uses the shared identity and pointer layout. +pub(crate) const SCHEDULER_RELEASE_OPCODE: u32 = 0x13; diff --git a/drivers/gpu/drm/asahi/g17/fw/m5/render.rs b/drivers/gpu/drm/asahi/g17/fw/m5/render.rs new file mode 100644 index 0000000000000..f53956102b610 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fw/m5/render.rs @@ -0,0 +1,643 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! M5 tiling and fragment command encoders, with shared kick bindings. + +#![allow(dead_code)] + +use super::super::kick::{KickTimestamp, RegisterWrite, RegisterArrayBinding}; + +use super::render_tail::{self as tail, RenderStage, RenderTail}; + +pub(crate) const M5_TA_COMMAND_BYTES: usize = 0x990; +pub(crate) const M5_3D_COMMAND_BYTES: usize = 0x2240; +pub(crate) const M5_TA_SELECTOR: u32 = 0; +pub(crate) const M5_3D_SELECTOR: u32 = 1; + +pub(crate) const M5_TA_PROGRAM_OFFSET: u32 = 0x60; + +pub(crate) const M5_3D_PROGRAM_OFFSETS: [u32; 4] = [0xa0, 0x7c0, 0xee0, 0x1600]; + +const PROGRAM_AREA: usize = 0x700; +const REGISTER_BYTES: usize = 12; +pub(crate) const M5_PROGRAM_CAPACITY: usize = PROGRAM_AREA / REGISTER_BYTES; + +const TA_DISPATCH_REGISTER: u32 = 0x1c898; + +const FRAGMENT_TILE_REGISTER: u32 = 0x10038; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum M5RenderCommandError { + ProgramTooLong, + ProgramOffset, + RceCount, + MissingDispatchMirror, + AddressOverflow, + InvalidCompletionState, + InvalidUsageHint, + Kick, + SharedRing, +} + +#[derive(Clone, Copy, Debug)] +pub(crate) struct M5ProgramRef<'a> { + pub registers: &'a [RegisterWrite], + + pub rce_count: u16, + pub command_offset: u32, +} + +#[derive(Clone, Copy, Debug)] +pub(crate) struct M5FragmentProgramRefs<'a> { + + pub main: M5ProgramRef<'a>, + + pub main_mid_count: u16, + pub store: M5ProgramRef<'a>, + pub resume: M5ProgramRef<'a>, + pub load: M5ProgramRef<'a>, +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(crate) struct M5StageCompletion { + + pub stamp_addresses: [u64; 2], + + pub stamp: u32, + + pub queue_index: u32, + + pub ordinal: u32, + + pub has_wait: bool, + + pub completion: u64, + + pub key: u64, +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(crate) struct M5HardwareStamp { + pub stamp: u32, + + pub entries: u32, + + pub generation: u64, + + pub base: u64, +} + +pub(crate) fn m5_hardware_stamp(p: &M5HardwareStamp) -> Result { + if p.entries == 0 { + return Err(M5RenderCommandError::InvalidCompletionState); + } + let value = (u64::from(p.stamp >> 8) | p.generation << 24).wrapping_sub(1); + let quotient = (value / u64::from(p.entries)) as u32; + let remainder = (value as u32).wrapping_sub(quotient.wrapping_mul(p.entries)); + let slot = (p.base as u32 & 0xff) + (remainder & 0xff); + let carry = slot / p.entries; + let index = (slot - carry * p.entries) & 0xff; + let phase = ((p.base >> 8) as u32) + .wrapping_add(quotient) + .wrapping_add(carry); + Ok(u64::from(index) | u64::from(phase) << 8) +} + +pub(crate) fn m5_render_completion( + p: &M5HardwareStamp, + pass: u8, +) -> Result { + Ok(m5_hardware_stamp(p)? | u64::from(pass) << 40) +} + +#[derive(Clone, Copy, Debug, Default)] +pub(crate) struct M5TaLinked3d { + + pub queue_index: u32, + + pub stamp: u32, + + pub flag: u8, + + pub completion: u64, +} + +#[derive(Clone, Copy, Debug, Default)] +pub(crate) struct M5TaPolicy { + pub policy_a: u32, + pub policy_b: u8, + pub policy_c: u32, +} + +#[derive(Clone, Copy, Debug, Default)] +pub(crate) struct M5ExtraAddresses { + + pub count: u32, + pub addresses: [u64; 4], + + pub policy: u8, +} + +#[derive(Clone, Copy, Debug, Default)] +pub(crate) struct M5TaCommandFields { + + pub sequence: u64, + pub completion: M5StageCompletion, + + pub pb_slot: u32, + + pub ta_word_a: u32, + + pub pb_state: u64, + + pub pm_scene: u64, + + pub scene_end_va: u64, + pub ta_word_b: u32, + pub ta_word_e: u32, + pub context_token: u64, + pub tail_descriptor: u64, + pub resolved_tail_value: u64, + pub tpc_va: u64, + pub tpc_size: u64, + pub linked_3d: M5TaLinked3d, + pub ta_word_c: u64, + pub ta_word_d: u64, + pub deflake_state_va: u64, + pub stage_flag_a: u8, + pub predecessor_id: u32, + + pub stage_word_a: u32, + pub stage_word_b: u32, + pub timestamp_va: u64, + + pub diagnostic_ta_timestamp_upper: u64, + pub ta_pair_a: [u64; 2], + + pub timestamp_counts: [u32; 2], + + pub ta_word_f: u64, + /// The shifted usage hint: the seven bytes of the tail's descriptor value after + /// its first byte. The byte after them starts the resolved usage pointer. + pub shifted_usage_hint: u64, + + pub uma_state: u64, + + pub tail_flag: u8, + pub pb_growth_request: u8, + pub shader_write_class: u8, + pub policy: M5TaPolicy, + pub extra: M5ExtraAddresses, +} + +#[derive(Clone, Copy, Debug, Default)] +pub(crate) struct M53dCommandFields { + pub sequence: u64, + pub completion: M5StageCompletion, + + pub pass: u8, + pub context_token: u64, + pub tail_descriptor: u64, + pub resolved_tail_value: Option, + pub fragment_word_l: u32, + + pub fragment_flag_a: bool, + + pub short_partial_programs: u8, + + pub tail_flag: u8, + + pub pm_state: u64, + + pub pm_scene: u64, + + pub scene_end_va: u64, + pub tilemap_va: u64, + pub msaa_control: u64, + pub fragment_word_a: u32, + pub sample_count: u32, + pub utile_grid: u32, + pub merge_bounds: [u64; 2], + pub fragment_word_b: u64, + pub tile_count: u32, + pub depth_bias_va: u64, + pub scissor_va: u64, + pub occlusion_query_va: u64, + pub fragment_word_c: u64, + + pub binding_pairs: [[u64; 2]; 23], + pub depth_dimensions: u64, + pub depth_va: u64, + pub depth_va_pair: [u64; 2], + pub stencil_va: u64, + pub stencil_va_pair: [u64; 2], + pub fragment_word_d: u64, + pub fragment_word_e: u64, + pub depth_stride: u64, + pub stencil_stride: u64, + pub depth_compression_va: u64, + pub depth_compression_stride: u64, + pub stencil_compression_va: u64, + pub stencil_compression_stride: u64, + pub sample_aux_flags: [u64; 2], + pub size_aux_control: [u64; 2], + pub eot_program: [u64; 2], + pub partial_eot_program: [u64; 2], + pub fragment_word_f: u64, + pub fragment_word_g: u32, + pub fragment_word_h: u64, + pub fragment_word_i: u32, + pub zs_clear_values: u64, + pub fragment_word_j: u64, + pub fragment_word_k: u32, + pub sample_size: u32, + pub fragment_pass_flag: u8, + pub fragment_word_n: u32, + pub fragment_flag_c: u8, + pub stage_flag_a: u8, + pub predecessor_id: u32, + + pub stage_word_a: u32, + pub stage_word_b: u32, + pub fragment_pair_a: [u64; 2], + + pub timestamp_counts: [u32; 2], + + pub fragment_word_m: u64, + + pub uma_state: u64, + + pub timestamp_va: [u64; 2], + + pub empty_tile_flags: [u8; 4], + pub fragment_flag_b: u8, + pub extra: M5ExtraAddresses, +} + +#[derive(Clone, Copy, Debug)] +pub(crate) struct M5RenderCommandInputs { + + pub context_id: u32, + + pub scheduler_record: u64, + pub ta: M5TaCommandFields, + pub fragment: M53dCommandFields, +} + +impl Default for M5RenderCommandInputs { + fn default() -> Self { + + Self { + context_id: 0, + scheduler_record: 0, + ta: M5TaCommandFields { stage_word_b: u32::MAX, ..Default::default() }, + fragment: M53dCommandFields { stage_word_b: u32::MAX, ..Default::default() }, + } + } +} + +fn put8(out: &mut [u8], offset: usize, value: u8) { + out[offset] = value; +} + +fn put16(out: &mut [u8], offset: usize, value: u16) { + out[offset..offset + 2].copy_from_slice(&value.to_le_bytes()); +} + +fn put32(out: &mut [u8], offset: usize, value: u32) { + out[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); +} + +fn put64(out: &mut [u8], offset: usize, value: u64) { + out[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); +} + +fn put128(out: &mut [u8], offset: usize, value: [u64; 2]) { + put64(out, offset, value[0]); + put64(out, offset + 8, value[1]); +} + +fn read32(out: &[u8], offset: usize) -> u32 { + u32::from_le_bytes(out[offset..offset + 4].try_into().unwrap()) +} + +fn completion_tail(out: &mut [u8], base: usize, c: &M5StageCompletion) { + put64(out, base, c.stamp_addresses[0]); + put64(out, base + 8, c.stamp_addresses[1]); + put32(out, base + 0x10, c.stamp); + put32(out, base + 0x14, c.queue_index); + put32(out, base + 0x18, c.ordinal); + put32(out, base + 0x1c, u32::from(c.has_wait)); +} + +/// Fill publication-time completion values without replacing the retained +/// resource, lifecycle or register-program fields of either command. +pub(crate) fn link_completions( + ta: &mut [u8; M5_TA_COMMAND_BYTES], + fragment: &mut [u8; M5_3D_COMMAND_BYTES], + kicks: [KickTimestamp; 2], + pass: u8, + fragment_busy: bool, +) { + put32(ta, 0x7a4, u32::from(fragment_busy)); + put64(ta, 0x7a8, kicks[1].completion_value(pass) & 0xff_ffff_ffff); + put8(ta, 0x7b0, pass); + put8(ta, 0x7b1, pass); + put64(ta, 0x8c6, kicks[0].completion_value(pass)); + put8(fragment, 0x90, pass); + put64(fragment, 0x2160, kicks[1].completion_value(pass)); +} + +/// Bind the command-state slot in each engine's completion tail. +pub(crate) fn bind_work_slot( + ta: &mut [u8; M5_TA_COMMAND_BYTES], + fragment: &mut [u8; M5_3D_COMMAND_BYTES], + slot: u32, +) -> Result<(), M5RenderCommandError> { + if slot >= 4 { return Err(M5RenderCommandError::InvalidCompletionState); } + put32(ta, 0x8be, slot); + put32(fragment, 0x2158, slot); + Ok(()) +} + +fn extra_tail(out: &mut [u8], stage: RenderStage, e: &M5ExtraAddresses) { + tail::encode_extra(out, stage, &tail::ExtraAddresses { + count: e.count, policy: e.policy, addresses: e.addresses, + }).expect("fixed render command extent"); +} + +fn check_program(p: &M5ProgramRef<'_>, offset: u32) -> Result<(), M5RenderCommandError> { + if p.command_offset != offset { + return Err(M5RenderCommandError::ProgramOffset); + } + if p.registers.len() > M5_PROGRAM_CAPACITY { + return Err(M5RenderCommandError::ProgramTooLong); + } + if usize::from(p.rce_count) > p.registers.len() || p.rce_count > 0x7f { + return Err(M5RenderCommandError::RceCount); + } + Ok(()) +} + +fn place_program(out: &mut [u8], low_alias: u64, p: &M5ProgramRef<'_>) -> Result { + let offset = p.command_offset as usize; + let address = low_alias + .checked_add(u64::from(p.command_offset)) + .ok_or(M5RenderCommandError::AddressOverflow)?; + put16(out, offset + PROGRAM_AREA + 8, 0); + put64(out, offset + PROGRAM_AREA, address); + for (index, register) in p.registers.iter().enumerate() { + let entry = offset + index * REGISTER_BYTES; + let header = (read32(out, entry) & 0xfffc0006) | (register.reg & 0x3fff9); + put32(out, entry, header); + put64(out, entry + 4, register.value); + } + put16(out, offset + PROGRAM_AREA + 8, p.registers.len() as u16); + Ok(address) +} + +/// The TA command's external-write class: whether the render's graphics +/// programs declare writes to memory, as the compiler reports it. +pub(crate) const fn shader_write_class(declared: bool) -> u8 { + // Forwarded as declared; the TA policy word is not inferred from it. + declared as u8 +} + +/// The shifted usage hint of a TA command, by programmed mesh class. The hint +/// covers only the seven bytes before the resolved usage pointer: what looks +/// like a high "pass rotation" bit in a wider read is that pointer's first +/// byte, not part of the hint. Other ordinary vertex programs have different +/// hints. +pub(crate) fn mesh_class_usage_hint(mesh_class: u8) -> u64 { + match mesh_class { + 0 => 0xc0c0, + 1 => 0x10ac0, + _ => 0x12b40, + } +} + +pub(crate) fn encode_m5_ta_command( + out: &mut [u8; M5_TA_COMMAND_BYTES], + low_alias: u64, + i: &M5RenderCommandInputs, + ta: &M5ProgramRef<'_>, +) -> Result<[Option; 4], M5RenderCommandError> { + check_program(ta, M5_TA_PROGRAM_OFFSET)?; + let dispatch = ta + .registers + .iter() + .find(|r| r.reg & !1 == TA_DISPATCH_REGISTER) + .ok_or(M5RenderCommandError::MissingDispatchMirror)? + .value; + low_alias.checked_add(u64::from(ta.command_offset)) + .ok_or(M5RenderCommandError::AddressOverflow)?; + let t = &i.ta; + let descriptor_value = tail::tiling_descriptor_value(t.tail_descriptor, t.shifted_usage_hint) + .ok_or(M5RenderCommandError::InvalidUsageHint)?; + put32(out, 0, M5_TA_SELECTOR); + put64(out, 4, t.sequence); + put32(out, 0xc, i.context_id); + tail::encode(out, RenderStage::Tiling, &RenderTail { + descriptor_value, resolved_value: Some(t.resolved_tail_value), + trailing_value: t.context_token, + }).expect("fixed TA command extent"); + put32(out, 0x8d6, t.ta_word_e); + put32(out, 0x48, t.ta_word_b); + put32(out, 0x18, t.pb_slot); + put32(out, 0x1c, t.ta_word_a); + put64(out, 0x20, t.pb_state); + put64(out, 0x28, t.pm_scene); + put64(out, 0x30, t.scene_end_va); + put64(out, 0x10, i.scheduler_record); + put64(out, 0x780, t.tpc_va); + put64(out, 0x788, t.tpc_size); + put32(out, 0x79c, t.linked_3d.queue_index); + put32(out, 0x7a0, t.linked_3d.stamp); + put32(out, 0x7a4, u32::from(t.linked_3d.flag & 1)); + put64(out, 0x7a8, t.linked_3d.completion & 0xff_ffff_ffff); + let pass = (t.linked_3d.completion >> 40) as u8; + put8(out, 0x7b1, pass); + put8(out, 0x7b0, pass); + put64(out, 0x7b6, t.ta_word_c); + put64(out, 0x7be, t.ta_word_d); + put64(out, 0x7d6, t.deflake_state_va); + put128(out, 0x7de, [0, 0]); + put64(out, 0x7ee, 0); + put32(out, 0x7b2, 0); + put32(out, 0x86a, u32::from(t.stage_flag_a & 1)); + put32(out, 0x86e, t.predecessor_id); + put32(out, 0x872, t.stage_word_a); + put32(out, 0x876, t.stage_word_b); + put128(out, 0x87a, [0, 0]); + put128(out, 0x90e, t.ta_pair_a); + put32(out, 0x91e, t.timestamp_counts[0]); + put32(out, 0x922, t.timestamp_counts[1]); + put64(out, 0x926, t.ta_word_f); + put64(out, 0x92e, t.uma_state); + put8(out, 0x936, t.tail_flag & 1); + put64(out, 0x8fe, t.timestamp_va); + put64(out, 0x906, t.diagnostic_ta_timestamp_upper); + put32(out, 0x892, u32::from(t.pb_growth_request & 1)); + put32(out, 0x89a, u32::from(t.shader_write_class & 1)); + put32(out, 0x89e, u32::from(m5_ta_policy_bit(&t.policy, t.shader_write_class))); + put32(out, 0x8a2, 0); + put64(out, 0x8f6, 0); + completion_tail(out, 0x8a6, &t.completion); + put64(out, 0x8c6, t.completion.completion); + put64(out, 0x8ce, t.completion.key); + + put32(out, 0x38, 0); + put16(out, 0x3c, 0); + put32(out, 0x40, 0); + let address = place_program(out, low_alias, ta)?; + put16(out, 0x38, ta.rce_count); + put16(out, 0x3a, ta.registers.len() as u16); + put16(out, 0x3c, ta.registers.len() as u16); + put64(out, 0x85e, dispatch); + extra_tail(out, RenderStage::Tiling, &t.extra); + let empty = None; + Ok([Some(RegisterArrayBinding::new(address, ta.rce_count as u8).map_err(|_| M5RenderCommandError::ProgramOffset)?), empty, empty, empty]) +} + +pub(crate) fn encode_m5_3d_command( + out: &mut [u8; M5_3D_COMMAND_BYTES], + low_alias: u64, + i: &M5RenderCommandInputs, + f: &M5FragmentProgramRefs<'_>, +) -> Result<[Option; 4], M5RenderCommandError> { + let programs = [f.main, f.store, f.resume, f.load]; + for (program, offset) in programs.iter().zip(M5_3D_PROGRAM_OFFSETS) { + check_program(program, offset)?; + low_alias.checked_add(u64::from(program.command_offset)) + .ok_or(M5RenderCommandError::AddressOverflow)?; + } + for partial in &programs[1..] { + if usize::from(partial.rce_count) != partial.registers.len() { + return Err(M5RenderCommandError::RceCount); + } + } + if f.main_mid_count < f.main.rce_count || usize::from(f.main_mid_count) > f.main.registers.len() { + return Err(M5RenderCommandError::RceCount); + } + let tile = f + .main + .registers + .iter() + .find(|r| r.reg & !1 == FRAGMENT_TILE_REGISTER) + .ok_or(M5RenderCommandError::MissingDispatchMirror)? + .value; + let t = &i.fragment; + put32(out, 0, M5_3D_SELECTOR); + put64(out, 4, t.sequence); + put32(out, 0xc, i.context_id); + tail::encode(out, RenderStage::Fragment, &RenderTail { + descriptor_value: t.tail_descriptor, resolved_value: t.resolved_tail_value, + trailing_value: t.context_token, + }).expect("fixed fragment command extent"); + put32(out, 0x2170, t.fragment_word_l); + put32(out, 0x2174, u32::from(t.fragment_flag_a)); + put32(out, 0x2100, u32::from(t.short_partial_programs & 1)); + put32(out, 0x20f8, 0); + + put128(out, 0x2178, [0, 0]); + put64(out, 0x2188, 0); + put8(out, 0x21d0, t.tail_flag & 1); + put64(out, 0x20, i.scheduler_record); + put64(out, 0x28, t.pm_state); + put64(out, 0x30, t.pm_scene); + put64(out, 0x38, t.scene_end_va); + put32(out, 0x58, t.fragment_word_a); + put64(out, 0x40, t.tilemap_va); + put64(out, 0x48, t.msaa_control); + put32(out, 0x50, t.sample_count); + put32(out, 0x54, t.utile_grid); + put128(out, 0x60, t.merge_bounds); + put64(out, 0x70, t.fragment_word_b); + put32(out, 0x78, t.tile_count); + put64(out, 0x1d30, t.scissor_va); + put64(out, 0x1d20, t.depth_bias_va); + put64(out, 0x1d40, t.occlusion_query_va); + for (index, pair) in t.binding_pairs.iter().enumerate() { + put128(out, 0x1d58 + index * 16, *pair); + } + put64(out, 0x1ec8, t.depth_va); + put64(out, 0x1ed0, t.depth_stride); + put64(out, 0x1ed8, t.depth_compression_stride); + put64(out, 0x1f28, t.fragment_word_d); + put128(out, 0x1ee0, t.depth_va_pair); + put64(out, 0x1ef0, t.depth_compression_va); + put64(out, 0x1ef8, t.stencil_va); + put64(out, 0x1f00, t.stencil_stride); + put64(out, 0x1f08, t.stencil_compression_stride); + put64(out, 0x1f30, t.fragment_word_e); + put128(out, 0x1f10, t.stencil_va_pair); + put64(out, 0x1f20, t.stencil_compression_va); + put64(out, 0x1f68, t.fragment_word_f); + put32(out, 0x1f70, t.fragment_word_g); + put128(out, 0x1f48, t.size_aux_control); + put128(out, 0x1f74, t.eot_program); + put64(out, 0x1f88, t.fragment_word_h); + put32(out, 0x1f90, t.fragment_word_i); + put128(out, 0x1f94, t.partial_eot_program); + put64(out, 0x1fa8, t.zs_clear_values); + put32(out, 0x1fb0, t.sample_size); + put128(out, 0x1f38, t.sample_aux_flags); + put64(out, 0x1d50, t.fragment_word_c); + put64(out, 0x1fc0, t.fragment_word_j); + put32(out, 0x1fc8, t.fragment_word_k); + put64(out, 0x1fd0, t.depth_dimensions); + out[0x1fdc..0x20dc].fill(0); + put32(out, 0x20dc, 0); + put128(out, 0x20e0, [0, 0]); + put32(out, 0x1fd8, 0); + put32(out, 0x20f0, 0); + put8(out, 0x90, t.pass); + put32(out, 0x20fc, u32::from(t.fragment_pass_flag & 1)); + put32(out, 0x2209, t.fragment_word_n); + put8(out, 0x220d, t.fragment_flag_c); + put32(out, 0x2104, u32::from(t.stage_flag_a & 1)); + put32(out, 0x2108, t.predecessor_id); + put32(out, 0x210c, t.stage_word_a); + put32(out, 0x2110, t.stage_word_b); + put128(out, 0x2114, [0, 0]); + put128(out, 0x21a8, t.fragment_pair_a); + put32(out, 0x21b8, t.timestamp_counts[0]); + put32(out, 0x21bc, t.timestamp_counts[1]); + put64(out, 0x21c0, t.fragment_word_m); + put64(out, 0x21c8, t.uma_state); + put128(out, 0x2198, t.timestamp_va); + completion_tail(out, 0x2140, &t.completion); + put64(out, 0x2160, t.completion.completion); + put64(out, 0x2190, 0); + for (index, flag) in t.empty_tile_flags.iter().enumerate() { + put32(out, 0x2124 + index * 4, u32::from(flag & 1)); + } + put32(out, 0x2138, u32::from(t.fragment_flag_b & 1)); + put64(out, 0x2168, t.completion.key); + extra_tail(out, RenderStage::Fragment, &t.extra); + + put32(out, 0x80, 0); + put16(out, 0x84, 0); + put32(out, 0x88, 0); + let mut rce = [None; 4]; + for (slot, program) in rce.iter_mut().zip(programs.iter()) { + let address = place_program(out, low_alias, program)?; + *slot = Some(RegisterArrayBinding::new(address, program.rce_count as u8).map_err(|_| M5RenderCommandError::ProgramOffset)?); + } + put64(out, 0x1f58, tile); + put64(out, 0x1f60, 0); + put16(out, 0x80, f.main.rce_count); + put16(out, 0x82, f.main_mid_count); + put16(out, 0x84, f.main_mid_count); + put16(out, 0x88, f.main.registers.len() as u16); + Ok(rce) +} + +fn m5_ta_policy_bit(p: &M5TaPolicy, shader_write_class: u8) -> bool { + if p.policy_a == 0 || !(1..=2).contains(&p.policy_b) || p.policy_c == 3 { + return false; + } + match p.policy_a { + 1 | 4 => shader_write_class & 1 == 0, + 3 => true, + _ => false, + } +} diff --git a/drivers/gpu/drm/asahi/g17/fw/m5/render_assembly.rs b/drivers/gpu/drm/asahi/g17/fw/m5/render_assembly.rs new file mode 100644 index 0000000000000..1966ab22f4878 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fw/m5/render_assembly.rs @@ -0,0 +1,87 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Assemble an M5 render pair from shared render resources. +use kernel::prelude::*; +use super::{render_registers as regs, render as cmd}; +use super::render_registers::{Workspace, M5RenderControls}; +use super::super::{render::RenderArgs, kick::{RegisterArrayBinding, RegisterWrite}}; + +pub(crate) const FRAGMENT_OFFSET: usize = 0x4000; +pub(crate) const COMMAND_STORAGE_BYTES: usize = 0x8000; + +pub(crate) struct EncodedRender { + pub(crate) rce: [[Option; 4]; 2], + pub(crate) register_counts: [u16; 5], + pub(crate) kick_flags: super::super::kick::StageKickFlags, +} + +fn commands(storage: &mut [u8]) -> Result<( + &mut [u8; cmd::M5_TA_COMMAND_BYTES], + &mut [u8; cmd::M5_3D_COMMAND_BYTES], +)> { + if storage.len() < COMMAND_STORAGE_BYTES { return Err(EINVAL); } + let (ta, fragment) = storage.split_at_mut(FRAGMENT_OFFSET); + Ok(((&mut ta[..cmd::M5_TA_COMMAND_BYTES]).try_into().map_err(|_| EINVAL)?, + (&mut fragment[..cmd::M5_3D_COMMAND_BYTES]).try_into().map_err(|_| EINVAL)?)) +} + +pub(crate) fn link_completions(storage: &mut [u8], + kicks: [super::super::kick::KickTimestamp; 2], pass: u8, fragment_busy: bool) -> Result { + let (ta, fragment) = commands(storage)?; + cmd::link_completions(ta, fragment, kicks, pass, fragment_busy); + Ok(()) +} + +pub(crate) fn bind_work_slot(storage: &mut [u8], slot: u32) -> Result { + let (ta, fragment) = commands(storage)?; + cmd::bind_work_slot(ta, fragment, slot).map_err(|_| EINVAL) +} + +#[inline(never)] +pub(crate) fn encode_into(storage: &mut [u8], low: u64, + args: &RenderArgs<'_>, controls: &M5RenderControls, + extra: regs::TaExtra, dispatch: regs::FragmentDispatch, + inputs: &cmd::M5RenderCommandInputs, workspace: &mut Workspace) + -> Result { + if storage.len() < COMMAND_STORAGE_BYTES { return Err(EINVAL); } + let fragment_low = low.checked_add(FRAGMENT_OFFSET as u64).ok_or(EINVAL)?; + fragment_low.checked_add(cmd::M5_3D_COMMAND_BYTES as u64).ok_or(EINVAL)?; + if inputs.context_id != u32::from(args.resources.context_id) { return Err(EINVAL); } + workspace.write(args, controls, extra, dispatch, inputs)?; + let short_partial = [RegisterWrite::new(0x160a8, 0x20), RegisterWrite::new(0x16068, 0x4_0000)]; + let (store, resume, load): (&[_], &[_], &[_]) = if inputs.fragment.short_partial_programs & 1 != 0 { + (&short_partial, &short_partial, &short_partial) + } else { (&workspace.store, &workspace.resume, &workspace.load) }; + let partial_counts = [store.len() as u16, resume.len() as u16, load.len() as u16]; + let count = workspace.ta.registers().len() as u16; + let ta_program = cmd::M5ProgramRef { registers: workspace.ta.registers(), + rce_count: count - 3, command_offset: cmd::M5_TA_PROGRAM_OFFSET }; + let view = |registers, rce_count, command_offset| cmd::M5ProgramRef { + registers, rce_count, command_offset }; + let fragment_programs = cmd::M5FragmentProgramRefs { + main: view(&workspace.fragment[..workspace.fragment_count], + workspace.fragment_count as u16 - 4, 0xa0), + main_mid_count: workspace.fragment_count as u16 - 2, + store: view(store, partial_counts[0], 0x7c0), + resume: view(resume, partial_counts[1], 0xee0), + load: view(load, partial_counts[2], 0x1600), + }; + let ta = &mut workspace.encoded_ta; + let fragment = &mut workspace.encoded_fragment; + ta.fill(0); fragment.fill(0); + let fragment_rce = cmd::encode_m5_3d_command(fragment, fragment_low, inputs, &fragment_programs).map_err(|_| EINVAL)?; + let ta_rce = cmd::encode_m5_ta_command(ta, low, inputs, &ta_program).map_err(|_| EINVAL)?; + let (ta_storage, fragment_storage) = storage.split_at_mut(FRAGMENT_OFFSET); + ta_storage[..cmd::M5_TA_COMMAND_BYTES].copy_from_slice(ta); + fragment_storage[..cmd::M5_3D_COMMAND_BYTES].copy_from_slice(fragment); + // A kick sets header bit 61 for a stage whose register program has the + // SCS bracket and bit 63 for its matrix bracket, from the same flags that + // gate those brackets. The fragment matrix bracket is never emitted here. + let kick_flags = super::super::kick::StageKickFlags { + scratch: [extra.scs_bracket & 1 != 0 || controls.ta.scs.scratch.enabled(), + controls.fragment.scs.scratch.enabled()], + matrix: [extra.matrix_bracket & 1 != 0, false], + }; + Ok(EncodedRender { rce: [ta_rce, fragment_rce], + register_counts: [count, workspace.fragment_count as u16, partial_counts[0], partial_counts[1], partial_counts[2]], + kick_flags }) +} diff --git a/drivers/gpu/drm/asahi/g17/fw/m5/render_demand.rs b/drivers/gpu/drm/asahi/g17/fw/m5/render_demand.rs new file mode 100644 index 0000000000000..d99970e2f126e --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fw/m5/render_demand.rs @@ -0,0 +1,290 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Program and tile storage pressure for shared render pools. Program sizes +//! are byte counts after linking; they are independent of register allocation. + +use super::pool::PoolError; +use super::pool_budget::Requirements; + +type Result = core::result::Result; + +fn add(a: u64, b: u64) -> Result { a.checked_add(b).ok_or(PoolError::Geometry) } +fn mul(a: u64, b: u64) -> Result { a.checked_mul(b).ok_or(PoolError::Geometry) } +fn count(value: u64) -> Result { value.try_into().map_err(|_| PoolError::Geometry) } +fn pages(value: u64, size: u64) -> Result { Ok(add(value, size - 1)? / size) } + +#[derive(Clone, Copy, Default)] +pub(crate) struct FragmentStorage { + pub(crate) primary_bytes: u32, + pub(crate) secondary_bytes: u32, + pub(crate) linked_bytes: u32, + pub(crate) minimum_primary_bytes: u32, + pub(crate) auxiliary_count: u32, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct FragmentParams { + pub(crate) pressure: u32, + pub(crate) page_units: u32, + pub(crate) large_units: u32, + pub(crate) auxiliary_units: u32, + pub(crate) primary_large: u32, + pub(crate) secondary_large: u32, +} + +impl FragmentStorage { + pub(crate) fn parameters(self) -> Result { + let primary = mul(add(u64::from(self.primary_bytes), u64::from(self.linked_bytes))? + .max(u64::from(self.minimum_primary_bytes)), 32)?; + let primary = add(primary, 0x4380)?; + let secondary = mul(u64::from(self.secondary_bytes), 32)?; + let primary_large = pages(primary, 0x10000)?; + let secondary_large = pages(secondary, 0x10000)?; + let small = add(pages(primary, 0x1000)?, pages(secondary, 0x1000)?)?; + let large = add(primary_large, secondary_large)?; + let auxiliary = mul(u64::from(self.auxiliary_count), 128)?; + let (auxiliary, reserved) = if auxiliary == 0 { (0, 0) } + else { (auxiliary | 64, auxiliary.max(0x4000) | 64) }; + let pressure = add(add(mul(small, 96)?, mul(large, 10)?)?, + add(mul(pages(auxiliary, 0x1000)?, 64)?, mul(pages(auxiliary, 0x10000)?, 5)?)?)?; + Ok(FragmentParams { + pressure: count(pressure)?, page_units: count(add(small, large)?)?, + large_units: count(large)?, + auxiliary_units: count(add(pages(reserved, 0x1000)?, pages(reserved, 0x10000)?)?)?, + primary_large: count(primary_large)?, secondary_large: count(secondary_large)?, + }) + } +} + +#[derive(Clone, Copy, Default)] +pub(crate) struct TileStorage { + pub(crate) threadgroup_bytes: u32, + pub(crate) imageblock_bytes: u32, + pub(crate) multisample_bytes: u32, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct TileParams { + pub(crate) pressure: u32, + pub(crate) page_units: u32, + pub(crate) large_units: u32, + pub(crate) threadgroup_large: u32, + pub(crate) imageblock_large: u32, + pub(crate) multisample_large: u32, +} + +impl TileStorage { + pub(crate) fn parameters(self, preempt_formula: bool) -> Result { + let imageblock = u64::from(self.imageblock_bytes); + let sizes = [u64::from(self.threadgroup_bytes), + if imageblock == 0 { 0 } else { add(imageblock, 64)? }, + u64::from(self.multisample_bytes)]; + let large = [pages(sizes[0], 0x10000)?, pages(sizes[1], 0x10000)?, + pages(sizes[2], 0x10000)?]; + let small = add(add(pages(sizes[0], 0x1000)?, pages(sizes[1], 0x1000)?)?, + pages(sizes[2], 0x1000)?)?; + let total_large = add(add(large[0], large[1])?, large[2])?; + let factor = if preempt_formula { 11 } else { 6 }; + Ok(TileParams { + pressure: count(add(mul(small, factor)?, mul(total_large, factor.min(8))?)?)?, + page_units: count(add(small, total_large)?)?, large_units: count(total_large)?, + threadgroup_large: count(large[0])?, imageblock_large: count(large[1])?, + multisample_large: count(large[2])?, + }) + } +} + +/// One submission's maximum pressure across all of its linked render +/// programs. Different programs may maximize different storage dimensions. +#[derive(Clone, Copy)] +pub(crate) struct RenderDemand { + counts: [u32; 5], + minimum_pages: u32, + ideal_pages: u32, + pressure: u32, + clusters: u64, + maximum: u64, + total: u64, + floor: u64, + limited_maximum: u64, + limited_total: u64, + included: bool, +} + +impl RenderDemand { + pub(crate) fn new(core_masks: &[u32], nominal_cores: u32, ptc_floor: u32) -> Result { + let clusters = core_masks.len() as u64; + let maximum = u64::from(core_masks.iter().map(|m| m.count_ones()).max().unwrap_or(0)); + let total = core_masks.iter().try_fold(0u64, |n, m| add(n, u64::from(m.count_ones())))?; + if total == 0 || ptc_floor == 0 || u64::from(nominal_cores) < total { + return Err(PoolError::Geometry); + } + let limit = mul(u64::from(nominal_cores), 2)? / clusters; + Ok(Self { + counts: [0; 5], minimum_pages: 0, ideal_pages: 0, pressure: 0, + clusters, maximum, total, floor: u64::from(ptc_floor), + limited_maximum: maximum.min(limit), limited_total: total.min(mul(limit, clusters)?), + included: false, + }) + } + + pub(crate) fn include_fragment(&mut self, fragment: FragmentParams, tile: TileParams) -> Result<()> { + let Self { clusters, maximum, total, floor, .. } = *self; + let base = add(mul(maximum, add(floor, 11)?)?, 58)?; + let fragment_small = u64::from(fragment.page_units); + let tile_small = u64::from(tile.page_units); + let pressure = add(u64::from(fragment.pressure), u64::from(tile.pressure))?; + let large = add(add(u64::from(fragment.large_units), u64::from(fragment.auxiliary_units))?, + u64::from(tile.large_units))?; + let parts = [fragment.primary_large, fragment.secondary_large, + tile.threadgroup_large, tile.imageblock_large, tile.multisample_large]; + let tails = parts[..4].iter().try_fold(0u64, |n, &v| add(n, u64::from(v)))?; + let ideal = add(add(add(base, u64::from(tile.multisample_large))?, tails)?, + add(mul(pressure.saturating_sub(floor), maximum)?, 64)?)?; + let minimum = add(add(add(base, u64::from(tile.multisample_large))?, tails)?, + add(add(mul(add(add(fragment_small, tile_small)?.saturating_sub(floor), tile_small)?, + maximum)?, mul(large, total)?)?, 64)?)?; + self.merge(parts, minimum, ideal, pressure, clusters) + } + + /// Vertex work has no imageblock and uses the tiling execution limits. + pub(crate) fn include_vertex(&mut self, storage: VertexStorage) -> Result<()> { + let params = FragmentStorage { + primary_bytes: storage.primary_bytes, secondary_bytes: storage.secondary_bytes, + linked_bytes: storage.linked_bytes, minimum_primary_bytes: storage.minimum_primary_bytes, + auxiliary_count: 0, + }.parameters()?; + let Self { clusters, floor, limited_maximum: maximum, limited_total: total, .. } = *self; + let large = u64::from(params.large_units); + let base = add(mul(maximum, add(floor, 11)?)?, 58)?; + let minimum = add(add(add(base, large)?, 64)?, + add(mul(u64::from(params.page_units).saturating_sub(floor), maximum)?, + mul(large, total)?)?)?; + let ideal = add(add(add(base, large)?, 64)?, + mul(u64::from(params.pressure).saturating_sub(floor), maximum)?)?; + self.merge([params.primary_large, params.secondary_large, 0, 0, 0], minimum, + ideal, u64::from(params.pressure), clusters) + } + + /// Compute storage is evaluated per dispatch before merging component maxima. + /// `resident_warps` and `shared_budget` are device policy, not API limits. + pub(crate) fn include_compute(&mut self, storage: ComputeStorage, + resident_warps: u32, shared_budget: u32) -> Result<()> { + if storage.threads == 0 || resident_warps == 0 { + return Err(PoolError::Geometry); + } + let frame = mul(add(u64::from(storage.primary_bytes), + u64::from(storage.linked_bytes))? + .max(u64::from(storage.minimum_primary_bytes)), 32)?; + let frame = add(frame, 0x4380)?; + let secondary = mul(u64::from(storage.secondary_bytes), 32)?; + let primary_large = pages(frame, 0x10000)?; + let secondary_large = pages(secondary, 0x10000)?; + let small = add(pages(frame, 0x1000)?, pages(secondary, 0x1000)?)?; + let large = add(primary_large, secondary_large)?; + let shared = u64::from(storage.threadgroup_bytes); + let shared_small = pages(shared, 0x1000)?; + let shared_large = pages(shared, 0x10000)?; + let warps = pages(u64::from(storage.threads), 32)?; + let mut groups = u64::from(resident_warps) / warps; + if shared != 0 { + let aligned_shared = mul(pages(shared, 64)?, 64)?; + groups = groups.min(u64::from(shared_budget) / aligned_shared); + } + if groups == 0 { return Err(PoolError::Geometry); } + let pressure = add(add(mul(mul(groups, warps)?.min(96), small)?, + mul(mul(groups, warps)?.min(10), large)?)?, + add(mul(groups, shared_small)?, mul(groups.min(8), shared_large)?)?)?; + let Self { clusters, maximum, total, floor, .. } = *self; + let group_large = mul(warps.min(10), large)?; + let tile = add(shared_small, shared_large)?; + let transient = add(add(mul(warps, small)?, group_large)?, tile)?; + let transient = add(transient.saturating_sub(floor), tile)?; + let base = add(mul(maximum, add(floor, 11)?)?, 58)?; + let tail = add(add(large, shared_large)?, 64)?; + let minimum = add(add(add(base, tail)?, mul(transient, maximum)?)?, + mul(add(shared_large, group_large)?, total)?)?; + let ideal = add(add(base, tail)?, mul(pressure.saturating_sub(floor), maximum)?)?; + self.merge([count(primary_large)?, count(secondary_large)?, + count(shared_large)?, 0, 0], minimum, ideal, pressure, clusters) + } + + /// Mesh/object execute on the tiling engine. Reuse compute's occupancy + /// arithmetic with the tiling-limited core topology, never compute's full + /// enabled core count. Resident/shared limits are explicit device policy. + pub(crate) fn include_mesh(&mut self, storage: ComputeStorage, + resident_warps: u32, shared_budget: u32) -> Result<()> { + let maximum = self.maximum; + let total = self.total; + self.maximum = self.limited_maximum; + self.total = self.limited_total; + let result = self.include_compute(storage, resident_warps, shared_budget); + self.maximum = maximum; + self.total = total; + result + } + + fn merge(&mut self, parts: [u32; 5], minimum: u64, ideal: u64, pressure: u64, + clusters: u64) -> Result<()> { + // Stored page counts are signed before conversion to byte counts. + // Reject an unrepresentable request instead of wrapping into a small pool. + let round = |value| -> Result { + let value = add(mul(value, clusters)?, 7)? & !7; + if value > i32::MAX as u64 { return Err(PoolError::Geometry); } + count(value) + }; + let mut next = *self; + next.minimum_pages = next.minimum_pages.max(round(minimum)?); + next.ideal_pages = next.ideal_pages.max(round(ideal)?); + next.pressure = next.pressure.max(count(pressure)?); + for (out, value) in next.counts.iter_mut().zip(parts) { *out = (*out).max(value); } + next.resource_flags()?; + next.included = true; + *self = next; + Ok(()) + } + + pub(crate) fn pressure(&self) -> u32 { self.pressure } + + pub(crate) fn resource_flags(&self) -> Result { + let [primary, secondary, shared, imageblock, multisample] = self.counts; + if primary > 0xff || secondary > 0xf || shared > 3 || imageblock > 0xff + || multisample > 0xff { return Err(PoolError::Geometry); } + Ok(0x14b | (u64::from(primary) << 10) | (u64::from(secondary) << 18) + | (u64::from(shared) << 22) | (u64::from(imageblock) << 24) + | (u64::from(multisample) << 32)) + } + + pub(crate) fn requirements(&self) -> Result { + if !self.included { + return Err(PoolError::Geometry); + } + let Self { maximum, clusters, .. } = *self; + let counts = self.counts.iter().try_fold(0u64, |n, &v| add(n, u64::from(v)))?; + let render_target = mul(add(add(mul(maximum, 11)?, counts)?, 10)?, clusters)?; + Ok(Requirements { + minimum_bytes: u64::from(self.minimum_pages) << 12, + ideal_bytes: u64::from(self.ideal_pages) << 12, + self_reserve_pages: 0, render_target_pages: count(render_target)?, has_spill: false, + }) + } +} + +#[derive(Clone, Copy, Default)] +pub(crate) struct VertexStorage { + pub(crate) primary_bytes: u32, + pub(crate) secondary_bytes: u32, + pub(crate) linked_bytes: u32, + pub(crate) minimum_primary_bytes: u32, +} + +/// Storage and execution width of a linked compute dispatch. +#[derive(Clone, Copy, Default)] +pub(crate) struct ComputeStorage { + pub(crate) primary_bytes: u32, + pub(crate) secondary_bytes: u32, + pub(crate) linked_bytes: u32, + pub(crate) minimum_primary_bytes: u32, + pub(crate) threadgroup_bytes: u32, + pub(crate) threads: u32, +} diff --git a/drivers/gpu/drm/asahi/g17/fw/m5/render_registers.rs b/drivers/gpu/drm/asahi/g17/fw/m5/render_registers.rs new file mode 100644 index 0000000000000..15e1887cee272 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fw/m5/render_registers.rs @@ -0,0 +1,635 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! M5 register adaptation over the shared render-pass/resource lowering. +#![allow(dead_code)] +use kernel::prelude::*; +use super::super::{kick::RegisterWrite, render::{RenderArgs, RenderPass, M5RegisterPrograms as RegisterPrograms}}; +use super::render::M5RenderCommandInputs; + +/// A pass's tiler context store for N clusters: 0x20 + 0x40 x N bytes of per-cluster control +/// words, the auxiliary sub-buffer at the next 64-byte boundary, the state stream +/// align64((N + 1) x 0x2e0) after it, and N x 0x680 + 0x800 bytes of the stream's storage +/// beyond it. Offsets from the store's base. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) struct ContextStore { + pub(crate) auxiliary: usize, + pub(crate) stream: usize, + pub(crate) extent: usize, +} +const fn align64(bytes: usize) -> usize { + (bytes + 63) & !63 +} +impl ContextStore { + pub(crate) const fn of(clusters: usize) -> Self { + let auxiliary = align64(0x20 + 0x40 * clusters); + let stream = auxiliary + align64(0x2e0 * (clusters + 1)); + Self { auxiliary, stream, extent: stream + 0x680 * clusters + 0x800 } + } + + /// `board`'s store: laid out for the G17D's own cluster count with its forms; other boards + /// keep the two-cluster store. + pub(crate) const fn selected(board: &super::initdata::Board) -> Self { + Self::of(if board.g17d_forms() { board.clusters as usize } else { 2 }) + } +} +const _: () = { + let two = ContextStore::of(2); + assert!(two.auxiliary == 0xc0 && two.stream == 0x980 && two.extent == 0x1e80); + let eight = ContextStore::of(8); + assert!(eight.auxiliary == 0x240 && eight.stream == 0x1c40 && eight.extent == 0x5840); +}; + +fn utile_config(p: &RenderPass) -> u64 { + (u64::from(p.utile_width / 16) << 12) | (u64::from(p.utile_height / 16) << 14) + | u64::from(p.samples_log2) +} +#[derive(Copy, Clone)] +pub(crate) struct M5RenderStageControls { + pub(crate) scs: crate::g17::render_resources::StageScratch, + + pub(crate) resource_flags: u64, + + pub(crate) resource_control: u64, + + pub(crate) ptc_max: u32, + pub(crate) work_key: u64, +} + +#[derive(Copy, Clone)] +pub(crate) struct M5RenderControls { + /// The board the render's encodings are for. + pub(crate) board: &'static super::initdata::Board, + /// Encoded event-command state for the initial fragment register list. + pub(crate) fragment_event_control: u64, + pub(crate) ta: M5RenderStageControls, + pub(crate) fragment: M5RenderStageControls, + pub(crate) ta_tile_control: u64, + + pub(crate) ta_dispatch_control: u64, + pub(crate) fragment_raster_control: u64, + + pub(crate) fragment_tile_flags: u64, + + pub(crate) store_pipeline_2_bind: u64, + pub(crate) store_pipeline_2: u64, + pub(crate) partial_store_pipeline_2_bind: u64, + pub(crate) partial_store_pipeline_2: u64, + pub(crate) partial_tile_flags: u64, + pub(crate) aux_fb_flags: u64, + pub(crate) aux_fb_control: u64, +} + +struct RegisterWriter<'a> { + values: &'a mut [RegisterWrite], + count: usize, +} + +impl RegisterWriter<'_> { + fn push(&mut self, number: u32, value: u64) -> Result { + let slot = self.values.get_mut(self.count) + .ok_or(EINVAL)?; + *slot = RegisterWrite { reg: number, value }; + self.count += 1; + Ok(()) + } + fn finish(self) -> Result { + if self.count != self.values.len() { + return Err(EINVAL); + } + Ok(()) + } +} + +fn value(registers: &[RegisterWrite], number: u32) -> Result { + registers + .iter() + .find(|r| r.reg == number) + .map(|r| r.value) + .ok_or(EINVAL) +} + +/// The event-command state of the initial fragment register list: 0 when +/// disabled, 3 for kind 1, otherwise 1. +pub(crate) const fn fragment_event_control(enabled: bool, kind: u32) -> u64 { + if !enabled { 0 } else if kind == 1 { 3 } else { 1 } +} + +const fn ptc_control(maximum: u32) -> u64 { + let maximum = if maximum == 0 { 0x60 } else { maximum }; + ((maximum as u64) << 32) | 0x400020 +} + +fn tile_state(p: &RenderPass, flags: u64) -> Result { + let tiles_x = (p.width as u64 + 31) / 32; + let tiles_y = (p.height as u64 + 31) / 32; + if !(1..=1024).contains(&tiles_x) || !(1..=1024).contains(&tiles_y) { + return Err(EINVAL); + } + + let geometry_mask = (0xfffff_u64 << 42) | (0xf_u64 << 38) | (1_u64 << 32); + if flags & geometry_mask != 0 { + return Err(EINVAL); + } + Ok(flags + | ((tiles_x - 1) << 42) + | ((tiles_y - 1) << 52) + | ((utile_config(p) & 0xf000) << 26) + | if p.layers > 1 { 1_u64 << 32 } else { 0 }) +} + +fn tile_blocks(pixels: u32) -> u64 { + let tiles = (pixels as u64 + 31) / 32; + let first = (((tiles + 3) / 4) + 3) & !3; + + (3 * first) | ((2 * first) << 10) | (first << 20) +} + +pub(crate) fn build_ta_registers_into( + p: &RenderPass, + controls: &M5RenderControls, + free_list_slot: u32, + out: &mut [RegisterWrite; 82], + common: &[RegisterWrite; 73], +) -> Result { + if controls.ta_dispatch_control & 1 != 0 { + return Err(EINVAL); + } + + tile_state(p, controls.fragment_tile_flags)?; + let mut out = RegisterWriter { values: out, count: 0 }; + for mut register in common.iter().copied() { + match register.reg { + 0x0a309 | 0x0a5a1 | 0x0d419 => continue, + // A fixed word of the TA program. Sample locations are bound by + // their own register. + 0x10131 => register.value = 0x88, + 0x1c8f8 => { + out.push(0x0d499, controls.ta.resource_flags)?; + out.push(0x0d419, controls.ta.resource_control)?; + out.push(0x0a5a1, ptc_control(controls.ta.ptc_max))?; + // Above four clusters the tiler's control word takes the topology term, as + // compute's dispatch control does; the fragment side's must agree with it. + register.value = + super::compute::dispatch_control(controls.board, controls.ta_tile_control); + } + 0x10171 => { + for (number, source) in [(0x1ca50, 0x101b9), (0x1ca58, 0x10149), (0x1ca60, 0x10141)] + { + out.push(number, value(common, source)?)?; + } + out.push(0x1ca70, tile_blocks(p.width))?; + out.push(0x1ca78, tile_blocks(p.height))?; + out.push(0x1ca68, value(common, 0x1c081)?)?; + } + 0x1c069 => register.value = tile_blocks(p.width), + 0x1c071 => register.value = tile_blocks(p.height), + 0x1c898 => { + if register.value > 1 { + return Err(EINVAL); + } + register.value |= controls.ta_dispatch_control; + } + 0x101c9 => register.value = controls.ta.scs.scratch.size_word(), + 0xd471 => register.value = u64::from(controls.ta.scs.auxiliary_allocation), + 0x1a0f1 => register.value = controls.ta.scs.scratch.stack_word(), + 0x1a099 => register.value = controls.ta.scs.intersection.words()[0], + 0x1a0a1 => register.value = controls.ta.scs.intersection.words()[1], + 0x1a069 => register.value = controls.ta.scs.intersection.words()[2], + 0x1a071 => register.value = controls.ta.scs.intersection.words()[3], + 0x1a0c9 => register.value = controls.ta.scs.intersection.words()[4], + 0x1a0d1 => register.value = controls.ta.scs.intersection.words()[5], + 0x10799 => { + register.value = controls.ta.scs.scratch.window_word() + | (register.value & (1 << 25)); + out.push(0x1a149, controls.ta.scs.scratch.driver_word())?; + }, + 0x10209 => { + out.push(0x0a739, 0)?; + register.value = controls.ta.work_key; + } + 0x1c9f0 | 0x14320 => register.value = controls.ta.work_key, + 0x14308 => register.value = free_list_slot as i32 as i64 as u64, + _ => {} + } + out.push(register.reg, register.value)?; + if register.reg == 0x01740 { + out.push(0x0a739, 0)?; + } + } + out.finish() +} + +pub(crate) fn build_fragment_registers_into( + p: &RenderPass, + controls: &M5RenderControls, + free_list_slot: u32, + out: &mut [RegisterWrite; 90], + common: &[RegisterWrite; 89], +) -> Result { + let geometry = tile_state(p, controls.fragment_tile_flags)?; + let mut out = RegisterWriter { values: out, count: 0 }; + let mut dispatch_writes = 0; + let mut tile_writes = 0; + for mut register in common.iter().copied() { + match register.reg { + 0x0a301 | 0x15109 | 0x15101 | 0x160a8 | 0x0a5a9 | 0x0d429 => continue, + 0x100b1 => register.reg = 0x16469, + 0x100d9 => register.reg = 0x16471, + 0x100b8 => { + register.reg = 0x16138; + register.value = super::compute::dispatch_control(controls.board, register.value); + } + 0x10791 => { + let scratch = controls.fragment.scs.scratch; + register.value = if scratch.enabled() { + (controls.fragment_raster_control | (1 << 24)) & 0x200001000204 + | (scratch.window_word() & 0xff0000) + | (controls.fragment_raster_control & (1 << 25)) + } else { controls.fragment_raster_control }; + }, + 0x101c1 => register.value = controls.fragment.scs.scratch.size_word(), + 0xd469 => register.value = u64::from(controls.fragment.scs.auxiliary_allocation), + 0x1a0f9 => register.value = controls.fragment.scs.scratch.stack_word(), + 0x1a0a9 => register.value = controls.fragment.scs.intersection.words()[0], + 0x1a0b1 => register.value = controls.fragment.scs.intersection.words()[1], + 0x1a079 => register.value = controls.fragment.scs.intersection.words()[2], + 0x1a081 => register.value = controls.fragment.scs.intersection.words()[3], + 0x1a0d9 => register.value = controls.fragment.scs.intersection.words()[4], + 0x1a0e1 => register.value = controls.fragment.scs.intersection.words()[5], + 0x15321 => { + out.push(0x0d491, controls.fragment.resource_flags)?; + out.push(0x0d429, controls.fragment.resource_control)?; + out.push(0x0a5a9, ptc_control(controls.fragment.ptc_max))?; + } + 0x16068 => { + tile_writes += 1; + if tile_writes == 1 { + register.value = geometry; + } + } + 0x16020 => { + dispatch_writes += 1; + if dispatch_writes == 3 { + if register.value != 0 { + return Err(EINVAL); + } + continue; + } + } + 0x160e0 => out.push(0x1a151, controls.fragment.scs.scratch.driver_word())?, + 0x10211 => { + out.push(0x0a741, controls.fragment_event_control)?; + register.value = controls.fragment.work_key; + } + 0x10420 => register.value = controls.fragment.work_key, + 0x14048 => register.value = free_list_slot as i32 as i64 as u64, + _ => {} + } + out.push(register.reg, register.value)?; + if register.reg == 0x15381 { + out.push(0x153b1, controls.store_pipeline_2_bind)?; + out.push(0x153b9, controls.store_pipeline_2)?; + } + if register.reg == 0x01731 { + out.push(0x0a741, 0)?; + } + } + if dispatch_writes != 3 || tile_writes != 2 { + return Err(EINVAL); + } + out.finish() +} + +#[derive(Copy, Clone)] +pub(crate) struct FragmentDispatch { + pub(crate) words: [u64; 2], +} + +impl FragmentDispatch { + + pub(crate) fn initial(self, descriptor_flag: u8) -> u64 { + if descriptor_flag & 1 != 0 { return 0x20; } + ((self.words[0] >> 27) & 0x1f_ffff_f860) + | (self.words[1].wrapping_shl(2) & 0x3_ffff_f87c) + | ((self.words[1] >> 32) << 7) + } +} + +#[derive(Copy, Clone)] +pub(crate) struct TaExtra { + pub(crate) gtp1: u64, + pub(crate) gtp2: u64, + pub(crate) gtp4: u64, + pub(crate) gtp5: u64, + pub(crate) gtp6: u64, + pub(crate) gtp7: u64, + pub(crate) segment_control: u64, + pub(crate) segment_index: u64, + pub(crate) min_free_tvb_pages: u64, + pub(crate) scs_bracket: u8, + pub(crate) matrix_bracket: u8, +} + +pub(crate) struct TaProgram { + registers: [RegisterWrite; 97], + length: usize, +} + +impl TaProgram { + pub(crate) const fn empty() -> Self { + Self { registers: [RegisterWrite {reg: 0, value: 0}; 97], length: 0 } + } + pub(crate) fn registers(&self) -> &[RegisterWrite] { &self.registers[..self.length] } + pub(crate) fn set_lifecycle(&mut self, value: u64) { + for r in &mut self.registers[..self.length] { + if matches!(r.reg, 0x1ca10 | 0x14a1 | 0xa349) { + r.value = value; + } + } + } + fn push(&mut self, number: u32, value: u64) -> Result { + let slot = self.registers.get_mut(self.length).ok_or(EINVAL)?; + *slot = RegisterWrite { reg: number, value }; + self.length += 1; + Ok(()) + } +} + +pub(crate) fn tiling_with_scratch_into(common: &[RegisterWrite], e: TaExtra, + mesh_storage: u64, out: &mut TaProgram) -> Result { + out.length = 0; + if e.scs_bracket & 1 != 0 { out.push(0x17e1, 1)?; } + if e.matrix_bracket & 1 != 0 { out.push(0x17f1, 1)?; } + let mut deflake = false; + for r in common.iter().copied() { + if r.reg == 0x1748 { continue; } + if r.reg == 0x1740 { + if e.scs_bracket & 1 != 0 { out.push(0x17d9, 1)?; } + if e.matrix_bracket & 1 != 0 { out.push(0x17e9, 1)?; } + continue; + } + out.push(r.reg, r.value)?; + let register = r.reg; + match register { + // In a mesh-object TA program the mesh rendering buffer register + // follows the scratch-window register. + 0x10799 if mesh_storage != 0 => out.push(0x1ca48, mesh_storage)?, + 0x1c9c8 => { out.push(0x1c041, e.gtp1)?; out.push(0x1c9d0, e.gtp1)?; } + 0x1c880 if !deflake => { + deflake = true; + for (n,v) in [(0x1c948,e.gtp4),(0x1c888,e.gtp5), + (0x1c890,e.segment_control),(0x1c918,e.segment_index)] { out.push(n,v)?; } + } + 0x1c9d8 => { + for (n,v) in [(0x1c089,e.gtp2),(0x1c9e0,e.gtp2), + (0x16c41,e.gtp6 & !0x7f),(0x1ca40,e.gtp6 & !0x7f), + (0x1c9a8,e.min_free_tvb_pages),(0x1c920,e.gtp7 & !0x3f)] { out.push(n,v)?; } + } + _ => {} + } + } + if out.length != 92 + usize::from(e.scs_bracket & 1) * 2 + usize::from(e.matrix_bracket & 1) * 2 + + usize::from(mesh_storage != 0) { + return Err(EINVAL); + } + Ok(()) +} + +pub(crate) fn fragment_main_with_scratch_into(p: &RenderPass, common: &[RegisterWrite], + dispatch: FragmentDispatch, descriptor_flag: u8, scs: bool, out: &mut [RegisterWrite]) -> Result { + + let mut n = 0; + let mut tile = 0; + for mut r in common.iter().copied() { + + if r.reg == 0x10051 { + r.value = isp_sample_control(p); + } + if r.reg == 0x16068 { + if tile == 2 { return Err(EINVAL); } + out[n] = RegisterWrite { reg: 0x160a8, + value: if tile == 0 { dispatch.initial(descriptor_flag) } else { 0x20 } }; + n += 1; + tile += 1; + } + let slot = out.get_mut(n).ok_or(EINVAL)?; + *slot = r; + n += 1; + // The SCS controls bracket the fragment main list after entry and exit. + if scs && matches!(r.reg, 0x1739 | 0x1731) { + *out.get_mut(n).ok_or(EINVAL)? = RegisterWrite { + reg: if r.reg == 0x1739 { 0x17e1 } else { 0x17d9 }, value: 1 }; + n += 1; + } + } + if n != out.len() || tile != 2 { return Err(EINVAL); } + Ok(()) +} + +/// ISP sample allocation, in the M5 register's units. +pub(crate) fn isp_sample_control(p: &RenderPass) -> u64 { + let samples = 1u32 << p.samples_log2; + if samples <= 1 { return 0; } + let sample_bytes = (u32::from(p.sample_size).max(8) + 7) & !7; + let groups = sample_bytes.div_ceil(32); + let units = (u32::from(p.utile_width) >> 3) * (u32::from(p.utile_height) >> 2) * groups; + let value = match samples { 2 => units.max(8) >> 3, 4 => units >> 1, 8 => units, _ => 0 }; + let value = if samples == 2 && p.utile_width == 16 && p.utile_height == 16 { value } else { value >> 1 }; + u64::from(value & 0x1f) << 16 +} + +fn partial(common: &[RegisterWrite], insert_after: u32, + c: &M5RenderControls, load: bool, out: &mut [RegisterWrite; N]) -> Result { + let mut w = RegisterWriter { values: out, count: 0 }; + for r in common.iter().copied() { + if r.reg == 0x15359 { continue; } + w.push(r.reg, r.value | if r.reg == 0x10039 { c.partial_tile_flags } else { 0 })?; + if r.reg == insert_after { + w.push(0x153b1, if load { c.store_pipeline_2_bind } else { c.partial_store_pipeline_2_bind })?; + w.push(0x153b9, if load { c.store_pipeline_2 } else { c.partial_store_pipeline_2 })?; + } + } + w.finish() +} + +/// Caller-owned assembly scratch. All resource addresses come from RenderArgs. +pub(crate) struct Workspace { + common: RegisterPrograms, + base: [RegisterWrite; 90], + pub(crate) ta: TaProgram, + pub(crate) fragment: [RegisterWrite; 94], + pub(crate) fragment_count: usize, + pub(crate) store: [RegisterWrite; 17], + pub(crate) resume: [RegisterWrite; 24], + pub(crate) load: [RegisterWrite; 12], + // Assemble in normal cached memory. Firmware command mappings on M5 + // are write-combined and must not be used for header read/modify/write. + pub(crate) encoded_ta: [u8; super::render::M5_TA_COMMAND_BYTES], + pub(crate) encoded_fragment: [u8; super::render::M5_3D_COMMAND_BYTES], +} +// SAFETY: Every field consists only of integer storage. +unsafe impl Zeroable for Workspace {} +impl Workspace { + #[inline(never)] + pub(crate) fn write(&mut self, args: &RenderArgs<'_>, controls: &M5RenderControls, + extra: TaExtra, dispatch: FragmentDispatch, inputs: &M5RenderCommandInputs) -> Result { + self.common.write_m5(args)?; + let r = args.resources; + let p = args.pass; + // The two sub-buffers sit at the context store's offsets from its base. + let base = r.deflake_va.checked_sub(0x10_0000_0000).ok_or(EINVAL)?; + let store = ContextStore::selected(controls.board); + let (state_offset, aux_offset) = (store.stream as u64, store.auxiliary as u64); + let mut vdm_seen = false; + for reg in &mut self.common.ta { + let register = reg.reg; + reg.value = match register { + 0x1c880 if vdm_seen => base.checked_add(state_offset).ok_or(EINVAL)?, + 0x10111 | 0x1c9b0 => base.checked_add(state_offset).ok_or(EINVAL)?, + 0x10119 | 0x1c9b8 => base.checked_add(aux_offset).ok_or(EINVAL)?, + 0x1c079 | 0x1c9d8 if p.layers == 1 => r.heapmeta_va.checked_sub(0x10_0000_0000).ok_or(EINVAL)?, + _ => reg.value, + }; + if reg.reg == 0x1c880 { vdm_seen = true; } + } + for reg in &mut self.common.fragment { + let register = reg.reg; + reg.value = match register { + 0x16060 if p.layers == 1 => r.heapmeta_va, + 0x15021 => controls.aux_fb_flags, + 0x15049 => controls.aux_fb_control, + // Main tile-control bits 17..19 carry the empty-tile and + // depth/stencil attachment action flags. The ordinary M5 + // profile has one stored colour attachment. An unstored ZLS + // aspect or a pass that skips empty + // tiles needs its corresponding action bit. Partial phases + // clear these bits independently and retain their own policy. + 0x10039 => { + let skip_empty = !p.process_empty_tiles; + reg.value + | (u64::from(skip_empty && inputs.fragment.empty_tile_flags[3] & 1 == 0) << 17) + | (u64::from(p.depth.va != 0 + && (p.zls_control & (1 << 15) != 0 || p.zls_control & (1 << 19) == 0)) << 18) + | (u64::from(p.stencil.va != 0 + && (p.zls_control & (1 << 14) != 0 || p.zls_control & (1 << 18) == 0)) << 19) + } + _ => reg.value, + }; + } + let mut c = *controls; + c.ta.work_key = super::super::work_key(r.context_id, args.ta.state_word); + c.fragment.work_key = super::super::work_key(r.context_id, args.fragment.state_word); + let ta: &mut [RegisterWrite;82] = (&mut self.base[..82]).try_into().map_err(|_| EINVAL)?; + build_ta_registers_into(p, &c, r.free_list_slot, ta, &self.common.ta)?; + let mut extra = extra; + extra.scs_bracket |= u8::from(c.ta.scs.scratch.enabled()); + tiling_with_scratch_into(ta, extra, r.mesh_storage, &mut self.ta)?; + self.ta.set_lifecycle((u64::from(inputs.ta.predecessor_id) << 32) | u64::from(inputs.ta.completion.key as u32)); + c.fragment_tile_flags = (c.fragment_tile_flags & !0x7ffff) | 0x36011 + | (u64::from(inputs.fragment.empty_tile_flags[3] & 1) << 18); + build_fragment_registers_into(p, &c, r.free_list_slot, &mut self.base, &self.common.fragment)?; + self.fragment_count = 92 + usize::from(c.fragment.scs.scratch.enabled()) * 2; + fragment_main_with_scratch_into(p, &self.base, dispatch, inputs.fragment.empty_tile_flags[3], + c.fragment.scs.scratch.enabled(), &mut self.fragment[..self.fragment_count])?; + for reg in &mut self.fragment { + if matches!(reg.reg, 0x160e0 | 0x1499 | 0xa341) { + reg.value = (u64::from(inputs.fragment.predecessor_id) << 32) | u64::from(inputs.fragment.completion.key as u32); + } + } + partial(&self.common.store, 0x15381, &c, false, &mut self.store)?; + // The ordinary SPM store takes its own empty-tile flag, independent + // of main empty-tile processing and of the SPM load's flag. Stored + // ZLS surfaces still force empty-tile processing in the store. + for register in &mut self.store { + if register.reg == 0x10039 { + let empty = inputs.fragment.empty_tile_flags[1] & 1 != 0 + || p.depth.va != 0 || p.stencil.va != 0; + register.value = (register.value & !(1 << 16)) + | (u64::from(empty) << 16); + } + } + partial(&self.common.resume, 0x15381, &c, false, &mut self.resume)?; + // The load phase uses the primary normal store binding. + // Normal EOT values are in the main array, before any adaptation. + c.store_pipeline_2_bind = self.common.fragment[2].value << 32; + c.store_pipeline_2 = self.common.fragment[3].value; + partial(&self.common.load, 0x15309, &c, true, &mut self.load)?; + // The SPM load phase takes its empty-tile policy from its own flag, + // which the fragment command also carries. It differs from the main + // and partial-store policy for multisample resolve passes. + for register in &mut self.load { + if register.reg == 0x10039 { + register.value = (register.value & !(1 << 16)) + | (u64::from(inputs.fragment.empty_tile_flags[2] & 1) << 16); + } + } + Ok(()) + } +} + +/// Populate command-side copies from the same pass that produces the registers. +/// Queue identity, completion sequencing and resource budgets remain caller-owned. +pub(crate) fn apply_pass(args: &RenderArgs<'_>, c: &M5RenderControls, + i: &mut M5RenderCommandInputs) -> Result { + let p=args.pass;let r=args.resources; + if p.width==0 || p.width>32768 || p.height==0 || p.height>32768 + || p.layers==0 || p.layers>2048 || r.context_id==0 || r.context_id>63 || p.samples_log2>3 + || !matches!((p.utile_width,p.utile_height),(32,32)|(32,16)|(16,16)) { return Err(EINVAL); } + let usc=|v:u32|p.usc_base.checked_add(u64::from(v&!7)).ok_or(EINVAL); + let spec=|v:u64|if p.rsrc_spec_hi {v}else{v&u64::from(u32::MAX)}; + // Check every address before changing command inputs. + let bg=usc(p.bg.offset)?;let pbg=usc(p.partial_bg.offset)?; + let eot=usc(p.eot_offset)?;let peot=usc(p.partial_eot_offset)?; + let deflake=r.deflake_va.checked_sub(0x10_0000_0000) + .and_then(|offset|offset.checked_add(ContextStore::selected(c.board).stream as u64)).ok_or(EINVAL)?; + i.context_id=u32::from(r.context_id); + i.ta.tpc_va=r.tpc_va;i.ta.tpc_size=r.tpc_size;i.ta.deflake_state_va=deflake; + // With the G17D forms the tiler asks the host for parameter-buffer blocks when the buffer + // runs out (a growth request, served by the shared growth worker) instead of rendering + // partially. J714 keeps 0. + i.ta.pb_growth_request = u8::from(c.board.g17d_forms()); + let f=&mut i.fragment; + f.empty_tile_flags[0]=u8::from(p.process_empty_tiles); + // The fragment command's pass flag: 1 in every ordinary render pass with the G17D forms. It + // follows the render context, not the attachments, sample count, layers or tile size. J714 + // keeps 0. + f.fragment_pass_flag = u8::from(c.board.g17d_forms()); + // The admitted ordinary single-sample profile has one stored colour + // attachment. Its clear-intent flag follows empty-tile intent; a LOAD + // pass uses zero here, while partial-store ZLS addresses can + // independently force empty tiles in the register program. + if p.samples_log2 == 0 { + f.empty_tile_flags[1] = u8::from(p.process_empty_tiles); + f.empty_tile_flags[2] = 0; + } else { + // Multisample stored-colour CLEAR and LOAD passes both process empty + // tiles in the SPM load. Only CLEAR sets the store flag; the BG + // resource specification publishes that intent in bit 6. An empty + // seed pass has no BG or empty-tile intent. + f.empty_tile_flags[1] = u8::from(p.process_empty_tiles && p.bg.rsrc_spec & (1 << 6) != 0); + f.empty_tile_flags[2] = u8::from(p.process_empty_tiles); + } + f.depth_bias_va=p.depth_bias_va;f.scissor_va=p.scissor_va;f.occlusion_query_va=p.occlusion_query_va; + f.binding_pairs[18]=[spec(p.bg.rsrc_spec),bg]; + f.binding_pairs[21]=[spec(p.partial_bg.rsrc_spec),pbg];f.binding_pairs[22][0]=p.zls_control; + f.depth_dimensions=p.depth_dimensions;f.depth_va=p.depth.va;f.depth_va_pair=[p.depth.va;2]; + f.stencil_va=p.stencil.va;f.stencil_va_pair=[p.stencil.va;2]; + f.depth_stride=p.depth.stride;f.stencil_stride=p.stencil.stride; + f.depth_compression_va=p.depth.compression_va;f.depth_compression_stride=p.depth.compression_stride; + f.stencil_compression_va=p.stencil.compression_va;f.stencil_compression_stride=p.stencil.compression_stride; + f.sample_aux_flags=[isp_sample_control(p),c.aux_fb_flags]; + f.size_aux_control=[u64::from(p.width)|(u64::from(p.height)<<32),c.aux_fb_control]; + f.eot_program=[spec(p.eot_rsrc_spec)<<32,eot]; + f.partial_eot_program=[spec(p.partial_eot_rsrc_spec)<<32,peot]; + f.zs_clear_values=u64::from(p.depth_clear)|(u64::from(p.stencil_clear|0x300)<<32); + f.tilemap_va=r.tilemap_va;f.msaa_control=p.msaa_control;f.sample_count=1u32< Option { + if context >= 64 || generation == 0 || generation >= (1u64 << 58) { + return None; + } + Some((u64::from(context) << 58) | generation) +} + +/// Merge the shifted usage hint into the TA tail's descriptor value. +/// +/// The value is eight bytes. Its first byte belongs to the descriptor; the +/// shifted hint fits only the following seven bytes. The next byte starts the +/// independent resolved usage pointer, so accepting an eighth hint byte would +/// overwrite that pointer. +pub(crate) fn tiling_descriptor_value(descriptor: u64, shifted_hint: u64) -> Option { + if shifted_hint > (u64::MAX >> 8) { + return None; + } + Some((descriptor & 0xff) | (shifted_hint << 8)) +} + +pub(crate) struct RenderTail { + + pub(crate) descriptor_value: u64, + + pub(crate) resolved_value: Option, + + pub(crate) trailing_value: u64, +} + +pub(crate) fn encode(out: &mut [u8], stage: RenderStage, value: &RenderTail) -> Option<()> { + let base = match stage { RenderStage::Tiling => 0x937, RenderStage::Fragment => 0x21d1 }; + let tail = out.get_mut(base..base + 24)?; + tail[..8].copy_from_slice(&value.descriptor_value.to_le_bytes()); + if let Some(resolved) = value.resolved_value { + tail[8..16].copy_from_slice(&resolved.to_le_bytes()); + } + tail[16..24].copy_from_slice(&value.trailing_value.to_le_bytes()); + Some(()) +} + +pub(crate) struct ExtraAddresses { + pub(crate) count: u32, + pub(crate) policy: u8, + pub(crate) addresses: [u64; 4], +} + +pub(crate) fn encode_extra(out: &mut [u8], stage: RenderStage, value: &ExtraAddresses) -> Option<()> { + let (base, end) = match stage { + RenderStage::Tiling => (0x96f, 0x990), + RenderStage::Fragment => (0x220e, 0x2233), + }; + let tail = out.get_mut(base..end)?; + tail[32] &= 0xf2; + if value.count != 0 || value.addresses.iter().any(|&address| address != 0) { + tail[32] |= 1 | ((value.policy & 3) << 2); + for (slot, address) in tail[..32].chunks_exact_mut(8).zip(value.addresses) { + slot.copy_from_slice(&address.to_le_bytes()); + } + } + if let RenderStage::Fragment = stage { tail[33..37].fill(0); } + Some(()) +} diff --git a/drivers/gpu/drm/asahi/g17/fw/m5/target.rs b/drivers/gpu/drm/asahi/g17/fw/m5/target.rs new file mode 100644 index 0000000000000..d03ecbfec77ec --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fw/m5/target.rs @@ -0,0 +1,71 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! M5 render-target suballocation geometry. + + +/// The part's minimum parameter-buffer size in 32 KiB pages. +pub(crate) const fn min_parameter_buffer_pages(clusters: u32) -> u64 { + 32 + 16 * clusters as u64 +} + +pub(crate) fn cluster_control(clusters: u32, pb_pages: u64) -> Option<[u64; 3]> { + let difference = match clusters { + 1 => return Some([1, 0, pb_pages]), + 2 => 4u64, + 4 => 5u64, + 8 => 6u64, + _ => return None, + }; + Some([0x40 | (difference << 8) | (difference << 19), 4, pb_pages]) +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct TargetSlices { + pub(crate) sizes: [usize; 8], + pub(crate) alignments: [usize; 8], + pub(crate) offsets: [usize; 8], + pub(crate) bytes: usize, +} + +impl TargetSlices { + pub(crate) fn new(tilemap_bytes: usize, tpc_used_bytes: usize, layers: u16, + kind: u32, segments: u32, pb_bytes: usize) -> Option { + if tilemap_bytes == 0 || tpc_used_bytes == 0 || layers == 0 || kind == 0 { + return None; + } + let mut sizes = [0; 8]; + let mut alignments = [0; 8]; + sizes[0] = tpc_used_bytes; + alignments[0] = 0x40; + if kind >= 2 { + if segments == 0 || pb_bytes == 0 { return None; } + let segments = segments as usize; + sizes[1] = tilemap_bytes.checked_mul(segments)?; + // Each segment's layer metadata, single-layer passes included. When the parameter + // buffer runs out and the firmware renders partially, the tiler continues each later + // segment k at this slice's base (named in the TA command) plus 0x100 k, and the + // firmware names that root in the next partial program. A single-layer pass's first segment + // takes its metadata from the target records instead, so the slice is used only once + // a pass splits: left unsized, its base was the compact window's first byte, and the + // segments' metadata landed in whatever lived there. + sizes[2] = 0x100usize.checked_mul(segments)?; + sizes[4] = (kind as usize).checked_mul(0x300)?.checked_add(0x680)?; + sizes[5] = (kind as usize).checked_mul(0x5c0)?; + + let pages = pb_bytes.checked_add(0x7fff)? / 0x8000; + let groups = (pages.checked_add(0x17ff)? / 0x1800).checked_add(1)?; + sizes[6] = groups.checked_mul(8)?.checked_mul(segments)?; + sizes[7] = 0x80; + alignments = [0x40, 0x40, 0x100, 0, 0x40, 0x40, 0x80, 0x40]; + } + let mut bytes = 0usize; + let mut offsets = [0; 8]; + for i in 0..8 { + if alignments[i] != 0 { + bytes = bytes.checked_add(alignments[i] - 1)? & !(alignments[i] - 1); + } + offsets[i] = bytes; + bytes = bytes.checked_add(sizes[i])?; + } + Some(Self { sizes, alignments, offsets, bytes }) + } +} diff --git a/drivers/gpu/drm/asahi/g17/fw/mod.rs b/drivers/gpu/drm/asahi/g17/fw/mod.rs index 1cb46fce2d602..df69458699b39 100644 --- a/drivers/gpu/drm/asahi/g17/fw/mod.rs +++ b/drivers/gpu/drm/asahi/g17/fw/mod.rs @@ -19,6 +19,9 @@ pub(crate) mod context; pub(crate) mod freelist; pub(crate) mod initdata; pub(crate) mod kick; +// Most M5 encodings are reached only once an M5 board has a page-table boot path. +#[allow(dead_code)] +pub(crate) mod m5; pub(crate) mod queue; pub(crate) mod render; pub(crate) mod render_rt; diff --git a/drivers/gpu/drm/asahi/g17/fw/queue.rs b/drivers/gpu/drm/asahi/g17/fw/queue.rs index 66e7d07db48a5..3de7282c2c020 100644 --- a/drivers/gpu/drm/asahi/g17/fw/queue.rs +++ b/drivers/gpu/drm/asahi/g17/fw/queue.rs @@ -386,8 +386,8 @@ pub(crate) struct TilingBinding { /// Queue whose configuration a [`QueueConfig`] carries. #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub(crate) enum QueueConfigTarget { - /// A tiling queue and its parameter-buffer binding. - Tiling(TilingBinding), + /// A tiling queue, optionally carrying a parameter-buffer binding. + Tiling(Option), /// A fragment queue. Fragment, /// A compute queue. @@ -513,7 +513,7 @@ impl QueueConfig { return Err(EINVAL); } let (data_master, pb_va, pb_token, pb_slot, fragment_qid) = match args.target { - QueueConfigTarget::Tiling(binding) => { + QueueConfigTarget::Tiling(Some(binding)) => { if binding.fragment_qid > QID_MAX { return Err(EINVAL); } @@ -525,6 +525,8 @@ impl QueueConfig { u32::from(binding.fragment_qid), ) } + QueueConfigTarget::Tiling(None) => + (DataMaster::Tiling, 0, u64::MAX, u32::MAX, u32::MAX), QueueConfigTarget::Fragment => (DataMaster::Fragment, 0, u64::MAX, u32::MAX, u32::MAX), QueueConfigTarget::Compute => (DataMaster::Compute, 0, u64::MAX, u32::MAX, u32::MAX), }; diff --git a/drivers/gpu/drm/asahi/g17/fw/render.rs b/drivers/gpu/drm/asahi/g17/fw/render.rs index 0e2f3284894ea..f084c65644239 100644 --- a/drivers/gpu/drm/asahi/g17/fw/render.rs +++ b/drivers/gpu/drm/asahi/g17/fw/render.rs @@ -243,6 +243,15 @@ pub(crate) struct RenderPass { pub(crate) few_primitives: bool, /// Host-only: the resolved barriers order the fragment kick, not the tiling kick. pub(crate) fragment_barriers: bool, + /// Host-only: the tiling kick waits on its own render and compute prefixes + /// (`Order::tiling_wait_through`), the fragment kick on the whole ones. M5 only. + pub(crate) stage_prefixes: bool, + /// Host-only: the render writes whole attachments: its tile stores reach no pixel + /// outside its own render area (hazard ordering). M5 only. + pub(crate) whole_attachments: bool, + /// A graphics program of the render writes external memory (buffer or image stores, + /// atomics); render-target outputs do not count. M5 only. + pub(crate) shader_writes_memory: bool, /// GPU address of the occlusion-query results, or zero. pub(crate) occlusion_query_va: u64, /// GPU address of the sampler heap, or zero. @@ -326,6 +335,10 @@ pub(crate) struct RenderResources { pub(crate) deflake_va: u64, /// GPU address of the tiling status word. pub(crate) ta_status_va: u64, + /// The TA mesh rendering buffer word for mesh/object kicks: the context VA + /// of the queue's mesh rendering buffer in the low 48 bits and its + /// 64 KiB chunk count above. Zero leaves the register out of the program. + pub(crate) mesh_storage: u64, /// GPU address of the fragment status word, as seen by the pass's VM. pub(crate) fragment_status_va: u64, /// Firmware alias of the fragment status word. @@ -472,7 +485,11 @@ struct PassWords { impl PassWords { fn new(pass: &RenderPass) -> Result { - if pass.samples_log2 > SAMPLES_LOG2_MAX { + Self::with_sample_limit(pass, SAMPLES_LOG2_MAX) + } + + fn with_sample_limit(pass: &RenderPass, max_samples_log2: u8) -> Result { + if pass.samples_log2 > max_samples_log2 { return Err(EINVAL); } let samples = 1u64 << pass.samples_log2; @@ -1001,6 +1018,97 @@ fn partial_load_registers( ] } +/// The M5 render register storage, shared by its phases. Keep this in the caller's heap +/// workspace when constructing descriptors. +pub(crate) struct M5RegisterPrograms { + pub(crate) ta: [RegisterWrite; TA_REGISTERS], + pub(crate) fragment: [RegisterWrite; FRAGMENT_REGISTERS], + pub(crate) store: [RegisterWrite; PARTIAL_STORE_REGISTERS], + pub(crate) resume: [RegisterWrite; PARTIAL_RESUME_REGISTERS], + pub(crate) load: [RegisterWrite; PARTIAL_LOAD_REGISTERS], +} +// SAFETY: All members contain only integer register writes. +unsafe impl Zeroable for M5RegisterPrograms {} +impl M5RegisterPrograms { + /// M5 partial phases add only their own depth/stencil direction while + /// preserving the pass's existing operations and surface layout bits. + pub(crate) fn write_m5(&mut self, args: &RenderArgs<'_>) -> Result { + // The M5 forms of the shared lowering: the A18 Pro arrays without the SCS enable + // (the M5 adaptation brackets its own scratch and intersection words), the + // one-cluster deflake layout, and up to eight samples. + let none = super::render_rt::RayQueryStage { + intersection: super::compute::IntersectionTables::NONE, + scratch: super::compute::ScratchRequest::new(0)?, + }; + let args = &RenderArgs { + programs: RegisterPrograms::G17P, + deflake: Deflake::ONE_CLUSTER, + ray_query: [none; 2], + ..*args + }; + let g = Geometry::new(args.pass)?; + let w = PassWords::with_sample_limit(args.pass, 3)?; + without_scs_enable(&mut self.ta, &ta_registers(args, &g, &w)?, TA_SCS_ENABLE_INDEX)?; + without_scs_enable( + &mut self.fragment, + &fragment_registers(args, &g, &w), + FRAGMENT_SCS_ENABLE_INDEX, + )?; + self.store = partial_store_registers(args.pass, &w); + self.resume = partial_resume_registers(args.pass, &w); + self.load = partial_load_registers(args.pass, &w); + // The partial phases start from the pass's ZLS word with both directions of each + // present surface, as the M5 lowering always has. + let mut partial = args.pass.zls_control; + if args.pass.depth.va != 0 { + partial |= ZLS_LOAD_DEPTH | ZLS_STORE_DEPTH; + } + if args.pass.stencil.va != 0 { + partial |= ZLS_LOAD_STENCIL | ZLS_STORE_STENCIL; + } + for array in [&mut self.store[..], &mut self.resume[..], &mut self.load[..]] { + for register in array { + if register.reg == 0x15319 { + register.value = partial; + } + } + } + const LOAD: u64 = (1 << 14) | (1 << 15); + const STORE: u64 = (1 << 18) | (1 << 19); + for register in &mut self.store { + if register.reg == 0x15319 { + register.value = (register.value & !LOAD) | (args.pass.zls_control & LOAD); + } else if register.reg == 0x10039 + && (args.pass.depth.va != 0 || args.pass.stencil.va != 0) + { + // A partial store forces empty tiles when either stored ZLS + // address (depth or stencil) exists. + register.value |= TILE_CONFIG_PROCESS_EMPTY_TILES; + } + } + for register in &mut self.load { + if register.reg == 0x15319 { + register.value = (register.value & !STORE) | (args.pass.zls_control & STORE); + } + } + Ok(()) + } +} + +/// Copies a lowered array without its SCS enable entry. +fn without_scs_enable( + out: &mut [RegisterWrite; N], + lowered: &[RegisterWrite; M], + index: usize, +) -> Result { + if M != N + 1 || lowered.get(index).is_none_or(|write| write.reg != SCS_ENABLE) { + return Err(EINVAL); + } + out[..index].copy_from_slice(&lowered[..index]); + out[index..].copy_from_slice(&lowered[index + 1..]); + Ok(()) +} + /// Tiling (TA) work descriptor (tag 0). #[repr(C, packed)] #[derive(Debug)] diff --git a/drivers/gpu/drm/asahi/g17/fw/render/storage.rs b/drivers/gpu/drm/asahi/g17/fw/render/storage.rs index e1e6a10949fb4..e8349d0fcd0e2 100644 --- a/drivers/gpu/drm/asahi/g17/fw/render/storage.rs +++ b/drivers/gpu/drm/asahi/g17/fw/render/storage.rs @@ -113,7 +113,8 @@ impl Support { } } -/// The viewport words consumed from the deflake page at +0x900. +/// Viewport words shared by the tiling engines; their location and control +/// value are selected by the firmware profile. #[repr(C)] #[derive(Copy, Clone)] pub(crate) struct Viewport { @@ -129,6 +130,11 @@ pub(crate) struct Viewport { scale_z: u32, } impl Viewport { + pub(crate) fn with_control(mut self, control: u32) -> Self { + self.unk_0 = control; + self + } + /// Refresh only defined words; firmware-owned gaps retain their contents. pub(crate) fn apply(&self, destination: &mut Self) { destination.unk_0 = self.unk_0; diff --git a/drivers/gpu/drm/asahi/g17/fw/render_rt.rs b/drivers/gpu/drm/asahi/g17/fw/render_rt.rs index 5ae23d84400eb..666209690efde 100644 --- a/drivers/gpu/drm/asahi/g17/fw/render_rt.rs +++ b/drivers/gpu/drm/asahi/g17/fw/render_rt.rs @@ -33,10 +33,12 @@ impl RayQueryStage { } } -pub(crate) fn required_blocks(active_cores: u32, pages: [u16; 2]) -> Result { +/// The render pool of an owner is shared by its `pairs` render pairs, each with +/// its own compute-pool extras. +pub(crate) fn required_blocks(active_cores: u32, pages: [u16; 2], pairs: u8) -> Result { let extra = compute_pool::extra_bytes(active_cores, pages[0])? .checked_add(compute_pool::extra_bytes(active_cores, pages[1])?) - .and_then(|bytes| bytes.checked_mul(crate::hw::t8140::queues::RENDER_PAIRS_PER_OWNER as u64)) + .and_then(|bytes| bytes.checked_mul(u64::from(pairs))) .ok_or(EOVERFLOW)?; let bytes = extra.checked_add((RENDER_BLOCKS * FREE_LIST_BLOCK_SIZE) as u64) .ok_or(EOVERFLOW)?; diff --git a/drivers/gpu/drm/asahi/g17/fw/variant.rs b/drivers/gpu/drm/asahi/g17/fw/variant.rs index 222473a8b8c36..8712322e976e0 100644 --- a/drivers/gpu/drm/asahi/g17/fw/variant.rs +++ b/drivers/gpu/drm/asahi/g17/fw/variant.rs @@ -64,6 +64,9 @@ pub(crate) struct Variant { pub(crate) few_primitives: bool, /// The firmware runs ray-traversal work (ray queries and intersection programs). pub(crate) ray_traversal: bool, + /// Renders may carry ray-query stages. The M5 command set has none: it encodes no render + /// ray-query state. + pub(crate) render_ray_query: bool, /// Tiling descriptors report whether the fragment queue still had earlier kicks /// outstanding (`fragment_busy`). pub(crate) fragment_busy: bool, @@ -77,6 +80,13 @@ pub(crate) struct Variant { pub(crate) compute: super::compute::ComputeProgram, pub(crate) opening_record: bool, pub(crate) split_firmware_area: bool, + /// Command validation applies the M5 command set: resource commands, the M5 render and + /// compute flags, 8x MSAA and the M5 tile-buffer limit. + pub(crate) m5_commands: bool, + /// The host holds the cores powered around a render pair's registration and its first + /// work. The bare-metal M5 firmware powers the cores for work by its own policy: under a + /// hold, the M5 Ultra's core-power witness reads unpowered until the hold is released. + pub(crate) registration_power_hold: bool, } /// A firmware build a SoC supports, identified by its loaded TEXT. @@ -117,6 +127,7 @@ pub(crate) const V26A428: Variant = Variant { mcache: true, few_primitives: true, ray_traversal: true, + render_ray_query: true, fragment_busy: true, lead_commands: false, programs: super::render::RegisterPrograms::G17P, @@ -124,6 +135,8 @@ pub(crate) const V26A428: Variant = Variant { compute: super::compute::ComputeProgram::G17P, opening_record: true, split_firmware_area: false, + m5_commands: false, + registration_power_hold: true, }; /// The M4 Pro firmware (25G83, `mBoot-18000.161.10`). @@ -146,6 +159,7 @@ pub(crate) const V25G83: Variant = Variant { // A fragment stage with the few-primitives flag hangs. few_primitives: false, ray_traversal: true, + render_ray_query: true, // When a tiling pass runs out of memory with this set, the firmware resumes it without // adding memory and the tiler stops silently; macOS always writes 0. fragment_busy: false, @@ -155,6 +169,17 @@ pub(crate) const V25G83: Variant = Variant { compute: super::compute::ComputeProgram::G17P, opening_record: true, split_firmware_area: false, + m5_commands: false, + registration_power_hold: true, +}; + +/// The 26A428 firmware of the M5 GPUs, which runs the M5 protocol (the M5 graph, queues and +/// command set). +pub(crate) const V26A428_M5: Variant = Variant { + m5_commands: true, + render_ray_query: false, + registration_power_hold: false, + ..V26A428 }; pub(crate) const V25F84: Variant = Variant { diff --git a/drivers/gpu/drm/asahi/g17/hazard.rs b/drivers/gpu/drm/asahi/g17/hazard.rs index 4d9e99d9a9cc0..071b0a15e3111 100644 --- a/drivers/gpu/drm/asahi/g17/hazard.rs +++ b/drivers/gpu/drm/asahi/g17/hazard.rs @@ -4,20 +4,24 @@ //! Tile stores can overlap even when the application's pixel regions do not. //! Keep independent attachments concurrent; order only conflicting live spans. //! -//! Within one VM, a render of another logical owner is ordered at publication: -//! the device mutex totally orders publications, and the later render's -//! fragment kick waits in firmware for the newest overlapping fragment kick of -//! every other pair. Admission never waits for such an overlap. Only a +//! Within one VM, a render of another logical owner, or of the same owner on +//! its other pair, is ordered at publication: the device mutex totally orders +//! publications, and the later render's fragment kick waits in firmware for the +//! newest overlapping fragment kick of every other pair. Admission never waits +//! for such an overlap. Only a //! cross-VM overlap through a shared object waits on the host for retirement, //! so that one VM's fault can never hold another VM's kick in firmware. use super::job::Order; +use kernel::prelude::kunit_tests; const MAX_SPANS: usize = 24; pub(crate) const WAIT_MS: u32 = 1000; /// Firmware barriers one render publication may add for same-VM overlaps, /// one per other pair. pub(crate) const PUBLICATION_BARRIERS: usize = 7; +/// Render pairs one logical owner can hold. +pub(crate) const MAX_OWNER_PAIRS: usize = crate::hw::t8140::queues::RENDER_PAIRS_MAX; #[derive(Clone, Copy)] struct Span { @@ -32,10 +36,13 @@ pub(crate) struct Footprint { spans: [Span; MAX_SPANS], count: usize, shared: bool, + /// The render writes whole attachments: its tile stores reach no pixel + /// outside its own render area. + whole: bool, } impl Footprint { - pub(crate) fn new(vm: u64) -> Self { + pub(crate) fn new(vm: u64, whole: bool) -> Self { Self { vm, spans: [Span { @@ -45,6 +52,7 @@ impl Footprint { }; MAX_SPANS], count: 0, shared: false, + whole, } } @@ -75,6 +83,12 @@ impl Footprint { if self.vm != other.vm && !(self.shared && other.shared) { return false; } + // Tile-granular stores of two whole-attachment renders write only + // pixels both renders own; unordered, they race as the application + // asked. + if self.whole && other.whole { + return false; + } self.spans[..self.count].iter().any(|a| { other.spans[..other.count] .iter() @@ -86,7 +100,11 @@ impl Footprint { #[derive(Clone, Copy, PartialEq, Eq)] pub(crate) enum Plan { Any, + /// Only this pair of the owner's. Pair(u8), + /// This pair of the owner's, else another of its pairs: the publication + /// barrier orders the overlap from either. + Join(u8), Wait, } @@ -108,6 +126,20 @@ pub(crate) struct Live<'a> { /// than `PUBLICATION_BARRIERS + 1` other pairs of its VM hold admitted renders: /// of any set of renders overlapping one publication, the last admitted saw all /// the others live. +/// +/// An owner's own uncovered overlaps on one of its pairs prefer that pair: the +/// pair's queues run its commands in publication order. On any other pair of +/// the owner's (it holds at most [`MAX_OWNER_PAIRS`]) the render's fragment +/// kick waits in firmware for that pair's newest overlapping fragment kick, as +/// for another owner's render ([`Pool::publication_barriers`] names it); tile +/// loads and stores happen only in fragment work. So with overlaps on two of +/// its pairs the render prefers the pair of the newer overlap, and a full +/// preferred pair moves the render to the owner's other pair instead of waiting. +/// That barrier is one more than the foreign ones: without a slot for it, a +/// render with overlaps on one pair stays on that pair, and one with overlaps +/// on two waits as a crowded publication does. +/// +/// [`Pool::publication_barriers`]: super::queue::admission::Pool::publication_barriers pub(crate) fn plan<'a>( owner: u64, prefix: Option, @@ -115,7 +147,9 @@ pub(crate) fn plan<'a>( footprint: &Footprint, live: impl IntoIterator>, ) -> Plan { - let mut selected = None; + // The owner's uncovered overlaps: (pair, newest sequence) for each of at + // most two of its pairs. + let mut own: [Option<(u8, u64)>; MAX_OWNER_PAIRS] = [None; MAX_OWNER_PAIRS]; let mut ordered = false; let mut pairs = [0u8; PUBLICATION_BARRIERS]; let mut count = 0; @@ -151,14 +185,93 @@ pub(crate) fn plan<'a>( if Order::contains(prefix, entry.sequence) { continue; } - match selected { - None => selected = Some(slot), - Some(prior) if prior == slot => {} - Some(_) => return Plan::Wait, + match own.iter_mut().find(|known| known.is_none_or(|(pair, _)| pair == slot)) { + Some(known) => { + let newest = known.map_or(entry.sequence, |(_, sequence)| sequence.max(entry.sequence)); + *known = Some((slot, newest)); + } + // More pairs than an owner can hold: not reachable. + None => return Plan::Wait, } } - if ordered && crowded { + // Foreign barriers come only with a foreign overlap, at most one per + // counted pair: the own cross-pair barrier needs a slot beside them. + let room = !(ordered && count == PUBLICATION_BARRIERS); + if ordered && crowded || own[1].is_some() && !room { return Plan::Wait; } - selected.map_or(Plan::Any, Plan::Pair) + match own { + [None, _] => Plan::Any, + [Some((slot, _)), None] if room => Plan::Join(slot), + [Some((slot, _)), None] => Plan::Pair(slot), + [Some(a), Some(b)] => Plan::Join(if a.1 >= b.1 { a.0 } else { b.0 }), + } +} + +#[kunit_tests(asahi_g17_hazard_plan)] +mod plan_tests { + use super::*; + + fn footprint(vm: u64, start: u64) -> Footprint { + let mut footprint = Footprint::new(vm, false); + footprint.push_vm(start, start + 0x1000); + footprint + } + + fn live(owner: u64, sequence: u64, slot: u8, footprint: &Footprint) -> Live<'_> { + Live { owner, sequence, slot: Some(slot), ticket: 0, footprint, quarantined: false } + } + + #[test] + fn an_own_overlap_prefers_its_pair() { + let mine = footprint(1, 0); + let earlier = footprint(1, 0x800); + assert!(plan(7, None, None, &mine, [live(7, 3, 4, &earlier)]) == Plan::Join(4)); + // A covered predecessor needs nothing. + assert!(plan(7, Some(3), None, &mine, [live(7, 3, 4, &earlier)]) == Plan::Any); + // Disjoint attachments stay free. + let other = footprint(1, 0x10000); + assert!(plan(7, None, None, &mine, [live(7, 3, 4, &other)]) == Plan::Any); + } + + #[test] + fn overlaps_on_two_own_pairs_prefer_the_newer_and_never_wait() { + let mine = footprint(1, 0); + let a = footprint(1, 0x800); + let b = footprint(1, 0x400); + assert!(plan(7, None, None, &mine, [live(7, 5, 2, &a), live(7, 9, 3, &b)]) == Plan::Join(3)); + assert!(plan(7, None, None, &mine, [live(7, 9, 2, &a), live(7, 5, 3, &b)]) == Plan::Join(2)); + assert!(plan(7, Some(5), None, &mine, [live(7, 5, 2, &a), live(7, 9, 3, &b)]) == Plan::Join(3)); + // A foreign same-VM overlap beside them is a barrier too. + assert!( + plan(7, None, None, &mine, [live(7, 5, 2, &a), live(7, 9, 3, &b), live(8, 1, 6, &b)]) + == Plan::Join(3) + ); + } + + #[test] + fn the_cross_pair_barrier_keeps_a_slot() { + let mine = footprint(1, 0); + let near = footprint(1, 0x800); + let far = footprint(1, 0x10000); + // Another owner of the VM holds `foreign` pairs, one with an overlap, + // then the owner's own overlaps on pair 2 and, if `across`, pair 3. + fn entries<'a>( + near: &'a Footprint, + far: &'a Footprint, + foreign: usize, + across: bool, + ) -> impl Iterator> + 'a { + (0..foreign) + .map(move |index| live(9, 1, 20 + index as u8, if index == 0 { near } else { far })) + .chain(core::iter::once(live(7, 5, 2, near))) + .chain(across.then(|| live(7, 6, 3, near))) + } + let full = PUBLICATION_BARRIERS; + assert!(plan(7, None, None, &mine, entries(&near, &far, full - 1, false)) == Plan::Join(2)); + assert!(plan(7, None, None, &mine, entries(&near, &far, full - 1, true)) == Plan::Join(3)); + // No slot left: one pair's overlap stays on that pair; two pairs' wait. + assert!(plan(7, None, None, &mine, entries(&near, &far, full, false)) == Plan::Pair(2)); + assert!(plan(7, None, None, &mine, entries(&near, &far, full, true)) == Plan::Wait); + } } diff --git a/drivers/gpu/drm/asahi/g17/initdata.rs b/drivers/gpu/drm/asahi/g17/initdata.rs index c0337c510e56b..5c333e05f8788 100644 --- a/drivers/gpu/drm/asahi/g17/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/initdata.rs @@ -12,11 +12,15 @@ use super::{ }; use crate::{driver::AsahiDrmDevice, hw::t8140, mmu}; use core::ops::Range; -use core::sync::atomic::{fence, AtomicU32, Ordering}; -use kernel::{c_str, prelude::*}; +use core::sync::atomic::{fence, AtomicU32, AtomicU64, Ordering}; +use kernel::{c_str, prelude::*, sync::Arc}; use t8140::{kernel_window as high, low}; mod v25g83; +mod m5; +mod work_pair; +pub(crate) mod outer_credit; +pub(crate) use outer_credit::{OuterClaim, OuterScope}; const EVENT_RING_RECHECK: u32 = 256; @@ -135,6 +139,7 @@ impl ComputeScheduler { /// Checked views used only within one publication call. Every fallible operation completes /// before either member of a paired render publication is copied into shared memory. struct WorkPublication<'a> { + published: Option<&'a AtomicU64>, record: &'a super::fw::channels::WorkSlot, destination: *mut u8, producer: &'a AtomicU32, @@ -153,10 +158,21 @@ impl WorkPublication<'_> { ); } } + + fn commit(self) -> u32 { + self.copy_record(); + fence(Ordering::SeqCst); + self.producer.store(self.next, Ordering::Relaxed); + if let Some(published) = self.published { published.fetch_add(1, Ordering::Release); } + fence(Ordering::SeqCst); + self.next + } } /// Session-long storage and mappings. Aliases are released before their backing objects. pub(super) struct InitData { + m5: bool, + outer_credits: Option>, // Live publication uses interior writes under the device mutex. This owner is Send but not // Sync, so safe host code cannot issue those writes concurrently through shared references. _single_publisher: core::marker::PhantomData>, @@ -169,9 +185,12 @@ pub(super) struct InitData { qos: usize, pb: usize, page_pool: usize, - compute_global: usize, - boot_control: usize, - boot_state: usize, + /// The G17P compute objects: the global state page and the boot free + /// list's control and state. M5 has none of them, and its build leaves + /// these `None`, so an unset handle can never name another object. + compute_global: Option, + boot_control: Option, + boot_state: Option, metrics_fw_va: u64, /// Client VA of the parameter-buffer metrics object (`Config::pm_metrics`). pm_metrics_va: u64, @@ -186,10 +205,12 @@ pub(super) struct InitData { compute_scheduler: Option, compute_active: core::cell::Cell, event_ring_checked: [core::cell::Cell<(u64, u64, u32)>; 2], - completion_rings: [usize; 2], + /// Completion-ring objects; a family sets only the rings it has. + completion_rings: [Option; 4], stamps: [usize; 2], - render_pages: usize, - render_runs: usize, + /// The G17P render page and run lists (client aliases). M5 has none. + render_pages: Option, + render_runs: Option, render_blocks: core::ops::Range, } @@ -262,6 +283,8 @@ impl InitData { /// Empty construction state, retained even if graph construction fails. pub(super) fn new() -> Self { Self { + m5: false, + outer_credits: None, _single_publisher: core::marker::PhantomData, aliases: KVec::new(), objects: KVec::new(), @@ -272,9 +295,9 @@ impl InitData { qos: 0, pb: 0, page_pool: 0, - compute_global: 0, - boot_control: 0, - boot_state: 0, + compute_global: None, + boot_control: None, + boot_state: None, metrics_fw_va: 0, pm_metrics_va: 0, recovery_slot_keys: None, @@ -285,10 +308,10 @@ impl InitData { compute_scheduler: None, compute_active: core::cell::Cell::new(false), event_ring_checked: [const { core::cell::Cell::new((0, 0, 0)) }; 2], - completion_rings: [0; 2], + completion_rings: [None; 4], stamps: [0; 2], - render_pages: 0, - render_runs: 0, + render_pages: None, + render_runs: None, render_blocks: 0..0, } } @@ -304,6 +327,18 @@ impl InitData { Ok(self.metrics_fw_va) } + /// Names the M5 renders' metrics object to the firmware through a kernel-window alias. + pub(super) fn bind_m5_metrics(&mut self, uat: &mmu::Uat, metrics: &KernelObject) -> Result { + if !self.m5 || self.metrics_fw_va != 0 { + return Err(EINVAL); + } + let alias = metrics.alias_in(uat.kernel_vm(), uat.kernel_dynamic_range(), + mmu::UAT_PGSZ as u64, mmu::PROT_FW_SHARED_RW)?; + self.metrics_fw_va = alias.iova(); + self.aliases.push(alias, GFP_KERNEL)?; + Ok(()) + } + pub(super) fn pb_descriptor_table(&self) -> Result<&KernelObject> { self.object(self.pb) } @@ -335,16 +370,25 @@ impl InitData { /// Initialize a fresh physical compute QID's row before exposing its queue. An installed /// or rebound queue keeps the row's firmware-owned counters and must never call this again. - /// Called for a fresh row or after its free-list release was consumed. - pub(super) fn initialize_compute_pool_blocks( - &self, - slot: u16, - page_list_va: u64, - blocks: usize, - ) -> Result { + /// Called for a fresh row or after its free-list release was consumed. The row has the + /// family's format: a G17P block row, or the M5 pool's descriptor. + pub(super) fn install_compute_pool(&self, descriptor: super::freelist::PoolDescriptor) -> Result { + use super::freelist::PoolDescriptor; + let slot = match descriptor { + PoolDescriptor::Neo { slot, .. } | PoolDescriptor::M5 { slot, .. } => slot, + }; if slot == 0 || slot > u16::from(super::fw::kick::QID_MAX) + 1 { return Err(EINVAL); } + match descriptor { + PoolDescriptor::Neo { slot, page_list, blocks } if !self.m5 => + self.initialize_compute_pool_blocks(slot, page_list, blocks), + PoolDescriptor::M5 { slot, words } if self.m5 => self.set_m5_uma(slot, words), + _ => Err(EINVAL), + } + } + + fn initialize_compute_pool_blocks(&self, slot: u16, page_list_va: u64, blocks: usize) -> Result { let entry = abi::PagePoolDescriptor::compute_blocks(page_list_va, blocks); let size = core::mem::size_of::(); let offset = usize::from(slot) * size; @@ -367,6 +411,53 @@ impl InitData { Ok(()) } + /// One M5 page-pool row: the pool's eight descriptor words at its 64-byte slot. + fn set_m5_uma(&self, slot: u16, words: [u64; 8]) -> Result { + let object = self.object(self.page_pool)?; + let offset = usize::from(slot) * core::mem::size_of::<[u64; 8]>(); + // Check the full row before the first store. + object.pointer(offset, core::mem::size_of::<[u64; 8]>())?; + for (index, word) in words.into_iter().enumerate() { + object.dword(offset + index * 8)?.store(word, Ordering::Relaxed); + } + fence(Ordering::SeqCst); + Ok(()) + } + + /// Whether a stopped session can prove its GPU engines retired, so that its pages may be + /// freed. M5 has no engine-retirement witness yet. + pub(super) fn has_engine_retirement_witness(&self) -> bool { + !self.m5 + } + + /// The statistics record tags the family's firmware emits. + pub(super) fn statistics_types(&self) -> u64 { + if self.m5 { + super::fw::m5::STATISTICS_TYPES + } else { + u64::from(super::fw::channels::STATISTICS_TYPES) + } + } + + /// A scheduler-state release record in the family's opcode. + pub(super) fn scheduler_release(&self, page_va: u64, id: [u8; 4]) -> super::fw::channels::SchedulerStateRelease { + let mut record = super::fw::channels::SchedulerStateRelease::new(page_va, id); + if self.m5 { + record.opcode = super::fw::m5::queue::SCHEDULER_RELEASE_OPCODE; + } + record + } + + /// Whether a control record of `opcode` holds the GPU powered until the firmware drains + /// it, by the family's rule. + pub(super) fn control_asserts_gpu_power(&self, opcode: super::fw::channels::DeviceControlOp) -> bool { + if self.m5 { + super::fw::m5::power::asserts_gpu_power(opcode as u32) + } else { + opcode.asserts_gpu_power() + } + } + pub(super) fn qos(&self) -> Result<&KernelObject> { self.object(self.qos) } @@ -395,7 +486,9 @@ impl InitData { if self.compute_shared.is_some() { return Ok(()); } - let state_va = self.object(self.boot_state)?.gpu_va(); + let boot_control = self.boot_control.ok_or(ENODEV)?; + let boot_state = self.boot_state.ok_or(ENODEV)?; + let state_va = self.object(boot_state)?.gpu_va(); let mut control = abi::FreeListControl::new(&abi::FreeListArgs { buffer_slot: u32::MAX, class: abi::FreeListClass::Two, @@ -405,9 +498,9 @@ impl InitData { state_va, }); control.run_cursor = COMPUTE_CONTROL_CURSOR; - self.object_mut(self.boot_control)?.bytes_mut().fill(0); - self.object_mut(self.boot_control)?.write(0, control)?; - self.object_mut(self.boot_state)?.bytes_mut().fill(0); + self.object_mut(boot_control)?.bytes_mut().fill(0); + self.object_mut(boot_control)?.write(0, control)?; + self.object_mut(boot_state)?.bytes_mut().fill(0); shared .pages .initialize::<[u64; low::RENDER_PAGE_LIST_SIZE / 8]>(0, |pages| { @@ -427,10 +520,14 @@ impl InitData { Ok(()) } + /// Whether `vm` has the shared compute aperture mapped. M5 has no shared + /// aperture: its compute queues need only their own pools, so every VM is ready. pub(super) fn compute_vm_cached(&self, vm: &mmu::Vm) -> bool { - self.compute_shared - .as_ref() - .is_some_and(|shared| vm.compute_pool_mappings_match(shared.id)) + self.m5 + || self + .compute_shared + .as_ref() + .is_some_and(|shared| vm.compute_pool_mappings_match(shared.id)) } pub(super) fn cache_compute_vm(&self, vm: &mmu::Vm) -> Result { @@ -460,7 +557,7 @@ impl InitData { } for slot in 0..2 { mappings.push( - self.object(self.completion_rings[slot])?.map_alias( + self.object(self.completion_rings[slot].ok_or(EINVAL)?)?.map_alias( vm, low::COMPLETION_RINGS[slot], mmu::PROT_GPU_SHARED_RW, @@ -479,7 +576,7 @@ impl InitData { low::PAGE_POOL_DESCRIPTORS, self.object(self.page_pool)?.gpu_va(), ]; - let base = abi::bundle::MAIN_CONFIG[Role::Primary as usize] + let base = self.main_config_offset(Role::Primary) + core::mem::offset_of!(abi::MainConfig, pb_descriptors_low_va); let bundle = self.object(self.bundle)?; let mut changed = false; @@ -500,7 +597,7 @@ impl InitData { return Ok(()); } let scheduler = self.compute_scheduler.as_ref().ok_or(EIO)?.object.gpu_va(); - let object = self.object(self.compute_global)?; + let object = self.object(self.compute_global.ok_or(ENODEV)?)?; let state = object .pointer(0, core::mem::size_of::())? .cast::(); @@ -526,15 +623,69 @@ impl InitData { Ok(()) } + /// The domain outer-ring waiters sleep on: unposted claims returned, or firmware + /// consumption seen by `poll_outer_progress`. Register only after dropping the device + /// mutex; a successful claim allocates no waiter fence. ENODEV off M5. + pub(crate) fn outer_available(&self) -> Result> { + Ok(self.outer_credits.as_ref().ok_or(ENODEV)?.available()) + } + + /// M5 TA, fragment and compute share the compute outer ring per priority. + /// The physical queue claim prevents reassignment while this credit lives. + pub(crate) fn reserve_outer(&self, priority: u8, count: u32) -> Result { + if !self.m5 { return Err(EINVAL); } + let cursors = self.work_cursors(priority, super::fw::queue::DataMaster::Compute)?; + self.outer_credits.as_ref().ok_or(EIO)?.reserve(priority, count, cursors) + } + + /// Hold the device mutex for this scope, including its Drop. The result + /// borrows only the claim, allowing the caller to publish through Firmware. + pub(crate) fn outer_scope<'a>(&self, claim: &'a mut OuterClaim) -> Result> { + self.outer_credits.as_ref().ok_or(EINVAL)?.scope(claim) + } + + pub(crate) fn outer_waiting(&self) -> bool { + self.outer_credits.as_ref().is_some_and(|credits| credits.waiting()) + } + + /// Return a notification for off-lock dispatch only on consumer progress. + /// Call while actual outer waiters exist; firmware may consume a config + /// without generating a GPU completion IRQ, so those waiters need polling. + pub(crate) fn poll_outer_progress(&self) -> Result>> { + let Some(credits) = self.outer_credits.as_ref().filter(|credits| credits.waiting()) else { + return Ok(None); + }; + let mut progressed = false; + for priority in 0..4 { + progressed |= credits.progressed(priority, + self.work_cursors(priority, super::fw::queue::DataMaster::Compute)?)?; + } + Ok(progressed.then(|| credits.available())) + } + + fn outer_capacity(&self, priority: u8, cursors: [u32; 3]) -> Result { + match &self.outer_credits { + Some(credits) => credits.capacity(priority, cursors), + None => outer_credit::free(cursors), + } + } + /// The sole host publisher holds the device mutex through the following /// inner and outer publication. Firmware consumers can only free capacity; /// neither this check nor waiting for room reserves or limits other queues. - /// `priority` selects the engine's ring of that priority class. + /// `priority` selects the engine's ring of that priority class. On M5, tiling, + /// fragment and compute share the compute ring, whose unclaimed credits this checks. pub(super) fn work_ready( &self, priority: u8, engine: super::fw::queue::DataMaster, ) -> Result { + use super::fw::queue::DataMaster; + if self.m5 { + let needed = if engine == DataMaster::Compute { 1 } else { 2 }; + return Ok(self.outer_capacity(priority, + self.work_cursors(priority, DataMaster::Compute)?)? >= needed); + } let [consumer, consumer_b, producer] = self.work_cursors(priority, engine)?; let next = (producer + 1) % super::fw::channels::WORK_RING_SLOTS; Ok(next != consumer && next != consumer_b) @@ -549,17 +700,23 @@ impl InitData { let engine = match record.data_master { 0 => DataMaster::Tiling, 1 => DataMaster::Fragment, - // Compute publications reserve their slots through the port. + // M5 compute is a shared-ring submission on the compute ring, published here + // under the device mutex; the classic compute path reserves through the port. + 2 if self.m5 => DataMaster::Compute, _ => return Err(EINVAL), }; let channel = channels::work_ring_index(priority, engine).ok_or(EINVAL)?; let [consumer, consumer_b, producer] = self.work_cursors(priority, engine)?; + if self.m5 && engine == DataMaster::Compute + && self.outer_capacity(priority, [consumer, consumer_b, producer])? < 1 { + return Err(EBUSY); + } let next = (producer + 1) % channels::WORK_RING_SLOTS; if next == consumer || next == consumer_b { return Err(EINVAL); } let destination = self.object(self.bundle)?.pointer( - abi::work_ring_offset(channel) + producer as usize * core::mem::size_of_val(record), + self.work_ring_offset(channel) + producer as usize * core::mem::size_of_val(record), core::mem::size_of_val(record), )?; let producer = self.object(self.cluster)?.word( @@ -567,7 +724,11 @@ impl InitData { + abi::work_state_offset(channel) + abi::ChannelState::PRODUCER, )?; + let published = if self.m5 && engine == DataMaster::Compute { + self.outer_credits.as_ref().map(|credits| credits.counter(priority)).transpose()? + } else { None }; Ok(WorkPublication { + published, record, destination, producer, @@ -602,6 +763,53 @@ impl InitData { Ok(values) } + /// Publishes an M5 compute record on the shared compute ring under the device mutex. + pub(super) fn publish_work( + &self, + priority: u8, + record: &super::fw::channels::WorkSlot, + ) -> Result { + Ok(self.work_publication(priority, record)?.commit()) + } + + /// Reserve both common-ring slots before touching either host port. The + /// returned views retain the graph borrow until the pair is published. + pub(super) fn m5_work_pair( + &self, + priority: u8, + fragment: &super::fw::channels::WorkSlot, + tiling: &super::fw::channels::WorkSlot, + ) -> Result> { + use super::fw::{channels, queue::DataMaster}; + if !self.m5 + || !matches!(fragment.data_master, 1 | 2) + || !matches!(tiling.data_master, 0 | 2) + { + return Err(EINVAL); + } + let cursors = self.work_cursors(priority, DataMaster::Compute)?; + if self.outer_capacity(priority, cursors)? < 2 { return Err(EBUSY); } + let window = work_pair::Window::new(cursors)?; + let channel = channels::work_ring_index(priority, DataMaster::Compute).ok_or(EINVAL)?; + let bundle = self.object(self.bundle)?; + let destinations = [window.start, window.next[0]].map(|index| { + bundle.pointer(self.work_ring_offset(channel) + + index as usize * size_of::(), size_of::()) + }); + let [fragment_destination, tiling_destination] = destinations; + let destinations = [fragment_destination?, tiling_destination?]; + let producer = self.object(self.cluster)?.word(self.state_offset(Role::Primary) + + abi::work_state_offset(channel) + abi::ChannelState::PRODUCER)?; + let copy = |slot: &channels::WorkSlot| channels::WorkSlot::new( + DataMaster::Compute, slot.queue_va, slot.qid, slot.items_end, slot.first != 0, + ); + let records = [copy(fragment)?, copy(tiling)?]; + // SAFETY: Complete, distinct slots were checked in the retained bundle; + // the device's single publisher holds its mutex across this borrow. + let published = self.outer_credits.as_ref().map(|credits| credits.counter(priority)).transpose()?; + Ok(unsafe { work_pair::Publication::new(records, destinations, producer, window, published) }) + } + /// Validate both destinations before exposing fragment, then tiling. The accounting token /// is committed after copying fragment; late tiling state is published before its slot. /// Deferred pairs retain both slot bodies without exposing either producer. @@ -620,6 +828,13 @@ impl InitData { { return Err(EINVAL); } + if self.m5 { + let pair = self.m5_work_pair(priority, fragment, tiling)?; + pair.prepare(); + commit_accounting()?; + publish_late_tiling(); + return Ok(if deferred { pair.next() } else { pair.publish() }); + } let fragment = self.work_publication(priority, fragment)?; let tiling = self.work_publication(priority, tiling)?; fragment.copy_record(); @@ -662,10 +877,11 @@ impl InitData { return Err(EBUSY); } let destination = self.object(self.bundle)?.pointer( - abi::work_ring_offset(channel) + current as usize * core::mem::size_of_val(record), + self.work_ring_offset(channel) + current as usize * core::mem::size_of_val(record), core::mem::size_of_val(record), )?; Ok(WorkPublication { + published: None, record, destination, producer, @@ -683,6 +899,12 @@ impl InitData { next: [u32; 2], ) -> Result { use super::fw::queue::DataMaster; + if self.m5 { + let pair = self.m5_work_pair(priority, fragment, tiling)?; + if pair.next() != next { return Err(EBUSY); } + pair.publish(); + return Ok(()); + } if fragment.data_master != DataMaster::Fragment as u32 || tiling.data_master != DataMaster::Tiling as u32 { @@ -716,7 +938,7 @@ impl InitData { |offset: usize| -> Result<*const AtomicU32> { Ok(state.word(base + offset)?) }; Ok(super::port::RingView { slots: self.object(self.bundle)?.pointer( - abi::work_ring_offset(channel), + self.work_ring_offset(channel), channels::WORK_RING_SLOTS as usize * core::mem::size_of::(), )?, consumers: [ @@ -742,9 +964,8 @@ impl InitData { qos: self .qos()? .pointer(0, core::mem::size_of::())?, - recovery: self - .object(self.cluster)? - .word(abi::private::STATUS_B + abi::PrimaryStatusB::RECOVERY_STATE)?, + // The word recovery_state() reads: the status block moves with the firmware build. + recovery: self.object(self.cluster)?.word(self.recovery_state_offset())?, mailbox, }) } @@ -763,7 +984,7 @@ impl InitData { seen: &mut [u64], mut record: impl FnMut(ComputeCompletion<'_>) -> Result, ) -> Result { - let object = self.object(self.completion_rings[0])?; + let object = self.object(self.completion_rings[0].ok_or(EINVAL)?)?; if seen.len() != COMPUTE_COMPLETION_SLOTS { return Err(EINVAL); } @@ -789,7 +1010,7 @@ impl InitData { seen: &mut [u64], mut record: impl FnMut(ComputeCompletion<'_>) -> Result, ) -> Result { - let object = self.object(self.completion_rings[0])?; + let object = self.object(self.completion_rings[0].ok_or(EINVAL)?)?; if seen.len() != COMPUTE_COMPLETION_SLOTS { return Err(EINVAL); } @@ -849,7 +1070,7 @@ impl InitData { active_cores: u32, variant: &super::fw::Variant, ) -> Result { - self.pm_metrics_va = cfg.pm_metrics.va; + self.pm_metrics_va = cfg.pm_metrics()?.va; self.recovery_slot_keys = cfg.recovery_slot_keys; let range = uat.geometry().kernel_range(); let env = BuildEnv { @@ -908,7 +1129,7 @@ impl InitData { }; use CpuMap::{WriteBack as WB, WriteCombined as WC}; self.cluster = self.add(env.fixed(high::PRIVATE_CLUSTER, cluster_size, FS, WC)?)?; - self.compute_global = self.add(env.fixed(high::ZERO_PAGE, mmu::UAT_PGSZ, FP, WC)?)?; + self.compute_global = Some(self.add(env.fixed(high::ZERO_PAGE, mmu::UAT_PGSZ, FP, WC)?)?); let pb = self.add(env.fixed(high::PB_DESCRIPTORS, mmu::UAT_PGSZ, FS, WC)?)?; self.object_mut(pb)? .initialize::(0, |table| { @@ -984,8 +1205,8 @@ impl InitData { let state = self.add(env.fixed(state, mmu::UAT_PGSZ, state_prot, WC)?)?; self.object_mut(state)?.write(0, initial)?; if class == abi::FreeListClass::Two { - self.boot_control = control; - self.boot_state = state; + self.boot_control = Some(control); + self.boot_state = Some(state); } } self.stamps[0] = self.add(env.fixed(high::DESCRIPTOR_PAGE_A, mmu::UAT_PGSZ, GFS, WC)?)?; @@ -1025,8 +1246,8 @@ impl InitData { WB, )?)?; } - self.render_pages = pages; - self.render_runs = runs; + self.render_pages = Some(pages); + self.render_runs = Some(runs); self.render_blocks = first_block..self.objects.len(); self.alias(pages, env.render, low::RENDER_PAGE_LIST, GS)?; self.alias(runs, env.render, low::RENDER_RUN_LIST, GS)?; @@ -1126,7 +1347,7 @@ impl InitData { for (slot, completion_ring) in completion_rings.iter_mut().enumerate() { let offset = high::BUNDLE + abi::bundle::COMPLETION_RINGS[slot] as u64; let ring = self.add(env.fixed(offset, abi::COMPLETION_RING_SIZE, FP, WB)?)?; - self.completion_rings[slot] = ring; + self.completion_rings[slot] = Some(ring); self.alias(ring, env.lower, low::COMPLETION_RINGS[slot], GS)?; *completion_ring = abi::CompletionRing { low_va: low::COMPLETION_RINGS[slot], @@ -1137,7 +1358,7 @@ impl InitData { self.qos = qos; fence(Ordering::SeqCst); let metrics_alias = - metrics.alias_in(env.vm, env.dynamic.clone(), cfg.pm_metrics.size as u64, GFS)?; + metrics.alias_in(env.vm, env.dynamic.clone(), cfg.pm_metrics()?.size as u64, GFS)?; self.metrics_fw_va = metrics_alias.iova(); self.aliases.push(metrics_alias, GFP_KERNEL)?; fence(Ordering::SeqCst); @@ -1178,7 +1399,7 @@ impl InitData { windows: cfg.windows, reserved_slots: cfg.reserved_slots, opps: &opps, - perf: cfg.perf, + perf: cfg.perf()?, completion_rings: bundle.completion_rings, qos_va: self.object(bundle.qos)?.gpu_va(), }; @@ -1186,7 +1407,7 @@ impl InitData { .initialize::(abi::bundle::HW_DATA, |hw| hw.init(&hw_args))?; self.object_mut(self.region_c)? .initialize::(0, |region| { - region.init(cfg.region_c); + region.init(cfg.region_c()?); Ok(()) })?; self.object_mut(self.cluster)? @@ -1308,7 +1529,7 @@ impl InitData { ) -> Result> { let mut mappings = KVec::with_capacity(24, GFP_KERNEL)?; mappings.push( - self.object_mut(self.render_pages)?.map_alias( + self.object_mut(self.render_pages.ok_or(EINVAL)?)?.map_alias( vm, low::RENDER_PAGE_LIST, mmu::PROT_GPU_SHARED_RW, @@ -1316,7 +1537,7 @@ impl InitData { GFP_KERNEL, )?; mappings.push( - self.object_mut(self.render_runs)?.map_alias( + self.object_mut(self.render_runs.ok_or(EINVAL)?)?.map_alias( vm, low::RENDER_RUN_LIST, mmu::PROT_GPU_SHARED_RW, @@ -1340,12 +1561,28 @@ impl InitData { Ok(mappings) } + pub(super) fn is_m5(&self) -> bool { + self.m5 + } + + /// The effort value the firmware is given for a requested `effort`. M5 publishes none while + /// work streams, except while a host thread waits for it (`waited`): then the full effort, + /// which the firmware takes as a floor for its performance state, so a submit-and-wait loop + /// runs at the top state. + pub(super) fn submission_effort(&self, effort: u32, waited: bool) -> u32 { + if self.m5 && !waited { 0 } + else { effort } + } + /// Writes the effort input in firmware order, then cleans its cache line to coherency. pub(super) fn publish_effort(&self, effort: u32) -> Result { let region = self.object(self.region_c)?; - let override_word = region.word(abi::RegionC::UNK_E8)?; - let effort_word = region.word(abi::RegionC::EFFORT)?; - let line = region.pointer(abi::RegionC::EFFORT, 4)?; + let (effort_offset, override_offset) = if self.m5 { + (super::fw::m5::power::EFFORT, super::fw::m5::power::EFFORT_OVERRIDE) + } else { (abi::RegionC::EFFORT, abi::RegionC::UNK_E8) }; + let override_word = region.word(override_offset)?; + let effort_word = region.word(effort_offset)?; + let line = region.pointer(effort_offset, 4)?; override_word.store(0, Ordering::Relaxed); effort_word.store(effort, Ordering::Relaxed); // SAFETY: line points into the retained kernel mapping; cleaning this host-owned cache @@ -1360,7 +1597,7 @@ impl InitData { pub(super) fn effort(&self) -> Result { Ok(self .object(self.region_c)? - .word(abi::RegionC::EFFORT)? + .word(if self.m5 { super::fw::m5::power::EFFORT } else { abi::RegionC::EFFORT })? .load(Ordering::Relaxed)) } @@ -1417,7 +1654,18 @@ impl InitData { Ok(opps) } + fn main_config_offset(&self, role: Role) -> usize { + if self.m5 { super::fw::m5::initdata::M5_MAIN_BUNDLE_OFFSETS[role as usize] } + else { abi::bundle::MAIN_CONFIG[role as usize] } + } + + fn work_ring_offset(&self, channel: usize) -> usize { + if self.m5 { super::fw::m5::initdata::M5_WORK_RING_BUNDLE_OFFSETS[channel] } + else { abi::work_ring_offset(channel) } + } + fn state_offset(&self, role: Role) -> usize { + if self.m5 { return super::fw::m5::initdata::M5_PRIVATE_STATE_GRIDS[role as usize]; } match role { Role::Primary => abi::private::PRIMARY_STATE, Role::Secondary => abi::private::SECONDARY_STATE, @@ -1425,6 +1673,7 @@ impl InitData { } fn status_offset(&self, role: Role) -> usize { + if self.m5 { return super::fw::m5::initdata::M5_PRIVATE_STATUS_A[role as usize]; } match role { Role::Primary => self.status_a, Role::Secondary => abi::private::SECONDARY_STATUS_A, @@ -1445,7 +1694,7 @@ impl InitData { cache.set((checked.0, checked.1, checked.2 - 1)); return Ok(()); } - let offset = abi::bundle::MAIN_CONFIG[role as usize] + let offset = self.main_config_offset(role) + core::mem::offset_of!(abi::MainConfig, channels) + abi::EVENT_CHANNEL * size_of::(); let bundle = self.object(self.bundle)?; @@ -1495,7 +1744,7 @@ impl InitData { abi::ChannelState::PRODUCER, ]) { *counter = state - .word(abi::CONTROL_STATE_OFFSET + offset)? + .word(self.state_offset(Role::Primary) + abi::CONTROL_STATE_OFFSET + offset)? .load(Ordering::Relaxed); } Ok(counters) @@ -1503,12 +1752,12 @@ impl InitData { pub(super) fn control_consumer(&self) -> Result<&AtomicU32> { self.object(self.cluster)? - .word(abi::CONTROL_STATE_OFFSET + abi::ChannelState::CONSUMER) + .word(self.state_offset(Role::Primary) + abi::CONTROL_STATE_OFFSET + abi::ChannelState::CONSUMER) } pub(super) fn control_producer(&self) -> Result<&AtomicU32> { self.object(self.cluster)? - .word(abi::CONTROL_STATE_OFFSET + abi::ChannelState::PRODUCER) + .word(self.state_offset(Role::Primary) + abi::CONTROL_STATE_OFFSET + abi::ChannelState::PRODUCER) } /// Publishes a control record under the device lock. All checked views are acquired before @@ -1520,13 +1769,15 @@ impl InitData { asserts_power: bool, next: u32, ) -> Result { - let offset = abi::bundle::MAIN_CONFIG[Role::Primary as usize] - + abi::MainConfig::CONTROL_RING - + (slot as usize).checked_mul(record.len()).ok_or(EINVAL)?; + let ring_base = if self.m5 { + super::fw::m5::initdata::M5_CONTROL_RING_BUNDLE_OFFSETS[0] + } else { self.main_config_offset(Role::Primary) + abi::MainConfig::CONTROL_RING }; + let offset = ring_base + (slot as usize).checked_mul(record.len()).ok_or(EINVAL)?; let ring = self.object(self.bundle)?.pointer(offset, record.len())?; let power = self .object(self.region_c)? - .pointer(abi::RegionC::POWER_ASSERT_TALLY, 4)?; + .pointer(if self.m5 { super::fw::m5::power::POWER_ASSERT_TALLY } + else { abi::RegionC::POWER_ASSERT_TALLY }, 4)?; let producer = self.control_producer()?; // SAFETY: The checked pointers remain valid for the lifetime of this non-Sync owner. // Firmware only reads records after the producer store and the device mutex serializes @@ -1566,13 +1817,42 @@ impl InitData { } impl InitData { + /// The M5 firmware's progress word ([`super::fw::m5::initdata::M5_PROGRESS`]); ENODEV on + /// other firmware. + pub(super) fn m5_progress(&self) -> Result { + use super::fw::m5::initdata as m; + if !self.m5 { + return Err(ENODEV); + } + fence(Ordering::Acquire); + Ok(self + .object(self.cluster)? + .word(m::M5_PRIVATE_LAYOUT.status_b + m::M5_PROGRESS)? + .load(Ordering::Relaxed)) + } + + fn recovery_state_offset(&self) -> usize { + if self.m5 { + super::fw::m5::initdata::M5_PRIVATE_LAYOUT.status_b + + super::fw::m5::initdata::M5_RECOVERY_STATE + } else { self.status_b + abi::PrimaryStatusB::RECOVERY_STATE } + } + /// Validate the serialized view before reading its retained host mapping. fn recovery_view_words(&self, view: usize, offset: usize) -> Result<[u32; N]> { let bundle = self.object(self.bundle)?; - let extent = *abi::recovery::VIEW_EXTENTS.get(view).ok_or(EINVAL)?; + let extent = if self.m5 { + *super::fw::m5::initdata::M5_BUNDLE_VIEW_EXTENTS.get(view).ok_or(EINVAL)? + } else { + *abi::recovery::VIEW_EXTENTS.get(view).ok_or(EINVAL)? + }; // Separately allocated views live at offset 0 of their own object. let (object, base) = match self.views { Some(views) => (self.object(*views.get(view).ok_or(EINVAL)?)?, 0), + None if self.m5 => ( + bundle, + *super::fw::m5::initdata::M5_BUNDLE_VIEW_OFFSETS.get(view).ok_or(EINVAL)?, + ), None => (bundle, *abi::bundle::VIEWS.get(view).ok_or(EINVAL)?), }; let end = offset @@ -1581,9 +1861,8 @@ impl InitData { if end > extent { return Err(EINVAL); } - let pointer_at = abi::bundle::MAIN_CONFIG[Role::Primary as usize] - + core::mem::offset_of!(abi::MainConfig, view_va) - + view * size_of::(); + let pointer_at = self.main_config_offset(Role::Primary) + + core::mem::offset_of!(abi::MainConfig, view_va) + view * size_of::(); fence(Ordering::Acquire); let pointer = bundle.read_words::<2>(pointer_at)?; let actual = u64::from(pointer[0]) | u64::from(pointer[1]) << 32; @@ -1595,11 +1874,26 @@ impl InitData { } impl super::recovery::Memory for InitData { + fn is_m5(&self) -> bool { + self.m5 + } fn fault_sources(&self) -> Result { + let mask_offset = if self.m5 { + super::fw::m5::initdata::M5_PRIVATE_LAYOUT.status_b + + super::fw::m5::initdata::M5_RECOVERY_SLOT_MASK + } else { + self.status_b + abi::PrimaryStatusB::RECOVERY_SLOT_MASK + }; let mask = self .object(self.cluster)? - .dword(self.status_b + abi::PrimaryStatusB::RECOVERY_SLOT_MASK)? + .dword(mask_offset)? .load(Ordering::Relaxed); + // M5's slot-identity storage is not available through this mapping. + // Keep firmware's explicit blame when attribution evidence is missing, + // while still allowing the recovery handshake to complete. + if self.m5 { + return Ok(super::recovery::Sources::sample(mask as u32, |_| None)); + } let mut sources = super::recovery::Sources::sample(mask as u32, |slot| { // Progress is independently available even when this firmware's key // address is unknown. Zero keeps ownership unknown and cannot override @@ -1625,21 +1919,26 @@ impl super::recovery::Memory for InitData { fence(Ordering::Acquire); Ok(self .object(self.cluster)? - .word(self.status_b + abi::PrimaryStatusB::RECOVERY_STATE)? + .word(self.recovery_state_offset())? .load(Ordering::Relaxed)) } fn host_recovery(&self) -> Result { Ok(self .object(self.cluster)? - .word(self.status_b + abi::PrimaryStatusB::HOST_RECOVERY)? + .word(if self.m5 { + super::fw::m5::initdata::M5_PRIVATE_LAYOUT.status_b + + super::fw::m5::initdata::M5_HOST_RECOVERY + } else { + self.status_b + abi::PrimaryStatusB::HOST_RECOVERY + })? .load(Ordering::Relaxed)) } fn transition(&self, expected: u32, next: u32) -> Result { let state = self .object(self.cluster)? - .word(self.status_b + abi::PrimaryStatusB::RECOVERY_STATE)?; + .word(self.recovery_state_offset())?; fence(Ordering::Acquire); if state.load(Ordering::Relaxed) != expected { return Err(EBUSY); @@ -1655,6 +1954,10 @@ impl super::recovery::Memory for InitData { // their own object. let (table, offset) = match self.views { Some(views) => (self.object(*views.get(4).ok_or(EINVAL)?)?, 0), + None if self.m5 => ( + self.object(self.bundle)?, + super::fw::m5::initdata::M5_BUNDLE_VIEW_OFFSETS[4], + ), None => (self.object(self.bundle)?, abi::bundle::VIEWS[4]), }; // Validate the complete table before publishing its first live word. @@ -1678,9 +1981,13 @@ impl super::recovery::Memory for InitData { return Err(EINVAL); } let cluster = self.object(self.cluster)?; - let offset = self.status_b - + abi::PrimaryStatusB::RECOVERY_INFO - + index * core::mem::size_of::(); + let base = if self.m5 { + super::fw::m5::initdata::M5_PRIVATE_LAYOUT.status_b + + super::fw::m5::initdata::M5_RECOVERY_INFO + } else { + self.status_b + abi::PrimaryStatusB::RECOVERY_INFO + }; + let offset = base + index * core::mem::size_of::(); fence(Ordering::Acquire); let flags = cluster .word(offset + core::mem::offset_of!(abi::RecoveryInfoEntry, flags))? @@ -1695,3 +2002,49 @@ impl super::recovery::Memory for InitData { kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); } } + +/// M5 thermal governor inputs and the host power budget it publishes. +impl InitData { + /// M5 only: the live host GPU power budget word of Region C. + pub(super) fn m5_power_budget(&self) -> Result { + if !self.m5 { return Err(EINVAL); } + Ok(self.object(self.region_c)?.word(super::thermal::REGION_C_POWER_BUDGET)?.load(Ordering::Relaxed)) + } + + /// M5 only: the GPU's current power draw in mW, as the firmware's power controller reports it + /// (integer word of the shared power-policy block). + pub(super) fn m5_gpu_power_mw(&self) -> Result { + if !self.m5 { return Err(EINVAL); } + Ok(self.object(self.bundle)? + .word(super::fw::m5::initdata::M5_SHARED_LAYOUT.policy + super::thermal::POLICY_GPU_POWER)? + .load(Ordering::Relaxed)) + } + + /// M5 only: whether the firmware's emergency clamp is active, from its primary performance + /// limiter in the shared power-policy block. + pub(super) fn m5_firmware_clamped(&self) -> Result { + if !self.m5 { return Err(EINVAL); } + let block = self.object(self.bundle)?; + let base = super::fw::m5::initdata::M5_SHARED_LAYOUT.policy; + let mask = block.word(base + super::thermal::POLICY_LIMITER_MASK)?.load(Ordering::Relaxed); + let limit = block.word(base + super::thermal::POLICY_LIMITER_LIMIT)?.load(Ordering::Relaxed); + Ok(super::thermal::is_emergency_clamp(mask, limit)) + } + + /// M5 only: publishes the host GPU power budget like the effort input. + pub(super) fn publish_m5_power_budget(&self, budget_mw: u32) -> Result { + if !self.m5 { return Err(EINVAL); } + let region = self.object(self.region_c)?; + let offset = super::thermal::REGION_C_POWER_BUDGET; + let word = region.word(offset)?; + let line = region.pointer(offset, 4)?; + word.store(budget_mw, Ordering::Relaxed); + // SAFETY: line points into the retained kernel mapping; cleaning this host-owned cache + // line and the following system barrier publish the naturally aligned store. + unsafe { + core::arch::asm!("dc cvac, {line}", "dsb sy", line = in(reg) line, + options(nostack, preserves_flags)) + }; + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/g17/initdata/m5.rs b/drivers/gpu/drm/asahi/g17/initdata/m5.rs new file mode 100644 index 0000000000000..ec150b54f1bee --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/initdata/m5.rs @@ -0,0 +1,229 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! M5 firmware graph backed by the shared G17 object and GPUVM owners. + +use super::*; +use crate::g17::fw::m5::initdata as m; + +const BASE: u64 = 0xffff_fc20_0000_0000; +const PRIVATE: u64 = BASE + 0x2_0000; +const REGION_C: u64 = BASE + 0x1a_0000; +const ROOTS: u64 = BASE + 0x1a_8000; +const REGION_A: u64 = BASE + 0x1b_8000; +const FWCTL: u64 = BASE + 0x1c_0000; +const QOS: u64 = BASE + 0x1c_8000; +const AON: u64 = BASE + 0x1d_0000; +const KICK_SHARED: u64 = BASE + 0x1d_8000; +const BUNDLE: u64 = BASE + 0xc078_8000; +const COMPLETION_HIGH: u64 = BASE + 0xc080_0000; +const COMPLETION_LOW: u64 = 0x70_0184_8000; +const COMPLETION_STRIDE: u64 = 0x2_8000; +const PB: u64 = BASE + 0x15e_0000; +const UMA: u64 = BASE + 0x15e_8000; + +/// Lower-root entry 1 of the M5 layout (Mesa's driver window), linked before the firmware starts +/// as the bare-metal J775D boot has always run. +const BOOT_LINK: u64 = 0x10_0008_0000; + +impl InitData { + /// The kernel lower-root ranges a bare-metal M5 firmware reaches, whose top-level links must + /// exist before it starts: entry 1, the descriptor tables and completion rings (entry 7) and + /// the first page of the canonical arena (`canonical`, entry 8), where queues go later. + pub(in crate::g17) fn m5_lower_links(canonical: core::ops::Range) + -> [core::ops::Range; 8] + { + let page = |address: u64| address..address + mmu::UAT_PGSZ as u64; + let ring = |index: u64| page(COMPLETION_LOW + index * COMPLETION_STRIDE); + [page(BOOT_LINK), page(m::REGION_VIEW_LOW_ADDRS[0]), page(m::REGION_VIEW_LOW_ADDRS[1]), + ring(0), ring(1), ring(2), ring(3), page(canonical.start)] + } + + /// Diagnostic-only read of an aligned word within the existing policy allocation. + /// Callers hold the device state mutex, retaining this mapping for the entire sample. + pub(crate) fn m5_policy_telemetry_word(&self, offset: usize) -> Result { + if !self.m5 || offset & 3 != 0 || offset.checked_add(4).is_none_or(|end| end > m::M5_POLICY_SIZE) { + return Err(EINVAL); + } + let at = m::M5_SHARED_LAYOUT.policy.checked_add(offset).ok_or(EINVAL)?; + Ok(self.object(self.bundle)?.word(at)?.load(Ordering::Relaxed)) + } + + pub(crate) fn note_m5_submission_history(&self) -> Result { + if !self.m5 { return Err(EINVAL); } + let private = self.object(self.cluster)?; + let base = m::M5_PRIVATE_LAYOUT.status_config; + for _ in 0..64 { + let generation = private.word(base + 0x74c)?.load(Ordering::Acquire); + if generation & 1 != 0 { core::hint::spin_loop(); continue; } + let config = private.read_words::<4>(base + 0x73c)?; + fence(Ordering::Acquire); + if generation != private.word(base + 0x74c)?.load(Ordering::Acquire) { + core::hint::spin_loop(); continue; + } + let period = u64::from(config[0]) | u64::from(config[1]) << 32; + let count = u64::from(config[2]) | u64::from(config[3]) << 32; + if period == 0 || count == 0 || count > 256 { return Err(EINVAL); } + let (counter, frequency) = crate::g17::qos::clock(); + let ticks = crate::g17::qos::time_from_host(counter, frequency)?; + // One slot per 25000 ticks of the period. + let index = (ticks % period) / 25_000; + if index >= count { return Err(ERANGE); } + let byte = private.pointer(base + 0x750 + index as usize, 1)?; + // SAFETY: retained, bounds-checked WC byte, written as a single byte + // after reading the even, stable generation, as the firmware expects. + unsafe { + let value = core::ptr::read_volatile(byte).wrapping_add(1); + core::ptr::write_volatile(byte, value); + }; + fence(Ordering::SeqCst); + return Ok(()); + } + Err(EBUSY) + } + + pub(crate) fn build_m5(&mut self, dev: &AsahiDrmDevice, uat: &mmu::Uat, + id: &crate::regs::G17Id, board: &m::Board) -> Result { + use mmu::{PROT_FW_SHARED_RW as RW, PROT_FW_SHARED_RO as RO, + PROT_GPU_SHARED_RW as GPU}; + self.m5 = true; + // TA, fragment and compute share the compute work ring per priority; every + // preparation's claim on it is counted from the first publication on. + self.outer_credits = Some(super::outer_credit::Credits::new()?); + let fixed = |address, size, protection| KernelObject::new(dev, + uat.kernel_vm(), Placement::At(address), size, protection, CpuMap::WriteCombined); + let fixed_controller = |address, size| { + KernelObject::new(dev, uat.kernel_vm(), Placement::At(address), size, + mmu::PROT_FW_PRIV_RW, CpuMap::WriteBack) + }; + let private = m::M5_PRIVATE_LAYOUT; + let shared = m::M5_SHARED_LAYOUT; + self.cluster = self.add(fixed(PRIVATE, private.total, RW)?)?; + self.region_c = self.add(fixed(REGION_C, mmu::UAT_PGSZ, RO)?)?; + self.roots = self.add(fixed(ROOTS, 0xc000, RO)?)?; + let region_a = self.add(fixed(REGION_A, mmu::UAT_PGSZ, RO)?)?; + self.add(fixed(FWCTL, mmu::UAT_PGSZ, RW)?)?; + self.qos = self.add(fixed(QOS, m::G17P_QOS_RESOURCE_SIZE, RW)?)?; + self.add(fixed(KICK_SHARED, m::M5_KICK_SHARED_SIZE, mmu::PROT_GPU_FW_SHARED_RW)?)?; + let aon = self.add(fixed_controller(AON, m::M5_AON_PTD_SIZE)?)?; + self.bundle = self.add(fixed_controller(BUNDLE, shared.total)?)?; + self.pb = self.add(fixed(PB, mmu::UAT_PGSZ, RW)?)?; + self.page_pool = self.add(fixed(UMA, mmu::UAT_PGSZ, RW)?)?; + for stage in 0..2 { + self.stamps[stage] = self.add(KernelObject::new(dev, uat.kernel_vm(), + Placement::In(uat.kernel_dynamic_range(), mmu::UAT_PGSZ as u64), + mmu::UAT_PGSZ, mmu::PROT_FW_SHARED_RW, CpuMap::WriteCombined)?)?; + } + self.alias(self.pb, uat.kernel_lower_vm(), m::REGION_VIEW_LOW_ADDRS[0], GPU)?; + self.alias(self.page_pool, uat.kernel_lower_vm(), m::REGION_VIEW_LOW_ADDRS[1], GPU)?; + let mut completions = [m::G17PKsmCompletionAliases { low: 0, high: 0 }; m::M5_25G76_COMPLETION_QUEUE_COUNT]; + for (index, pair) in completions.iter_mut().enumerate() { + pair.low = COMPLETION_LOW + index as u64 * COMPLETION_STRIDE; + pair.high = COMPLETION_HIGH + index as u64 * COMPLETION_STRIDE; + // After initialization the host only reads completion records. + // Match the standard G17 completion ring's CPU write-back mapping; + // keep firmware/GPU aliases, size and lifetime unchanged. + let backing = if index == 0 + { + KernelObject::new(dev, uat.kernel_vm(), Placement::At(pair.high), + m::M5_25G76_COMPLETION_BACKING_SIZE, RW, CpuMap::WriteBack)? + } else { + fixed(pair.high, m::M5_25G76_COMPLETION_BACKING_SIZE, RW)? + }; + let object = self.add(backing)?; + *self.completion_rings.get_mut(index).ok_or(EINVAL)? = Some(object); + self.alias(object, uat.kernel_lower_vm(), pair.low, GPU)?; + } + m::encode_common26_aon_ptd(board, &mut self.object_mut(aon)?.bytes_mut()[..m::M5_AON_PTD_SIZE]) + .map_err(|_| EINVAL)?; + m::encode_m5_region_c(self.object_mut(self.region_c)?.bytes_mut()).map_err(|_| EINVAL)?; + m::encode_m5_private_cluster(PRIVATE, FWCTL, + &mut self.object_mut(self.cluster)?.bytes_mut()[..private.total]).map_err(|_| EINVAL)?; + let windows = board.register_windows(); + let windows = &windows[..board.register_window_count()]; + for (index, window) in windows.iter().enumerate() { + let spec = board.register_window_spec(index).ok_or(EINVAL)?; + let stride = (u64::from(window.size) + mmu::UAT_PGMSK as u64) & !(mmu::UAT_PGMSK as u64); + for (instance, physical) in spec.instances.iter().copied().enumerate() { + let offset = physical & mmu::UAT_PGMSK as u64; + let address = window.device_va.checked_sub(offset).ok_or(EINVAL)? + + instance as u64 * stride; + let size = (offset + u64::from(window.size) + mmu::UAT_PGMSK as u64) + & !(mmu::UAT_PGMSK as u64); + self.aliases.push(uat.kernel_vm().map_io(dev.as_ref().as_ref(), address, + (physical & !(mmu::UAT_PGMSK as u64)).try_into()?, size.try_into()?, + mmu::PROT_FW_MMIO_RW)?, GFP_KERNEL)?; + } + } + let addresses = m::SharedAddresses { + region_a: self.object(region_a)?.gpu_va(), region_c: REGION_C, + hw_data_bundle: BUNDLE, hw_data_bundle_alloc: shared.total, + }; + let views = m::PrimaryRegionViews { + sentinel_high: BASE + 0xc07e_0000, alias_high: [PB, UMA], + }; + m::encode_m5_hw_data(board, completions, QOS, windows, + &mut self.object_mut(self.bundle)?.bytes_mut()[..m::M5_HW_DATA_SIZE]).map_err(|_| EINVAL)?; + m::set_m5_kick_shared( + &mut self.object_mut(self.bundle)?.bytes_mut()[..m::M5_HW_DATA_SIZE], KICK_SHARED) + .map_err(|_| EINVAL)?; + let masked_cores: u32 = id.core_masks.iter().map(|mask| mask.count_ones()).sum(); + let enabled_cores = if masked_cores != 0 { masked_cores } + else { id.num_clusters.checked_mul(id.num_cores).ok_or(EOVERFLOW)? }; + m::set_m5_enabled_cores( + &mut self.object_mut(self.bundle)?.bytes_mut()[..m::M5_HW_DATA_SIZE], enabled_cores) + .map_err(|_| EINVAL)?; + m::encode_m5_bundle_policy(m::M5PolicyBindings { + state: PRIVATE + private.policy_state as u64, aon_ptd: AON, boot_timestamp: 0, + }, self.object_mut(self.bundle)?.bytes_mut()).map_err(|_| EINVAL)?; + // The controller configuration applies where the part's leakage calibration matches the + // one it was made for, or on a board whose configuration holds for every part. + let calibrated = match board.calibration { + None => true, + Some(expected) => { + let node = dev.as_ref().as_ref().of_node().ok_or(EIO)?; + let calibration: Result> = + node.get_property(c_str!("apple,leakage-calibration-words")); + calibration.as_ref().is_ok_and(|words| words.as_slice() == expected) + } + }; + if let Some(defaults) = board.controller_defaults.filter(|_| calibrated) { + defaults(self.object_mut(self.bundle)?.bytes_mut()).map_err(|_| EINVAL)?; + let maximum = self.object(self.bundle)?.word(shared.policy + m::M5_POLICY_MAXIMUM_POWER)? + .load(Ordering::Relaxed); + m::set_m5_region_maximum_power(self.object_mut(self.region_c)?.bytes_mut(), maximum) + .map_err(|_| EINVAL)?; + } + if let Some(microseconds) = board.idle_standby_us { + m::set_m5_region_idle_standby(self.object_mut(self.region_c)?.bytes_mut(), microseconds) + .map_err(|_| EINVAL)?; + } + for (index, role) in [m::InstanceRole::Primary, m::InstanceRole::Secondary].into_iter().enumerate() { + let instance = m::InstanceAddresses { + root: ROOTS + index as u64 * 0x8000, + main_config: BUNDLE + shared.main_config[index] as u64, + status_a: PRIVATE + private.roles[index].status_a as u64, + status_b: if index == 0 { PRIVATE + private.status_b as u64 } else { 0 }, + secondary_extras: if index == 0 { [0; 2] } else { + [PRIVATE + private.status_config as u64, PRIVATE + private.secondary_extra_1 as u64] + }, + }; + let profile = m::RootProfile::J714G17S26A428; + let channels = m::derive_channel_table_for_profile(profile, role, + PRIVATE + m::M5_PRIVATE_STATE_GRIDS[index] as u64, + instance.status_a, instance.main_config, BUNDLE); + let offset = shared.main_config[index]; + m::encode_main_config_for_profile(profile, role, &addresses, &instance, + &channels, if index == 0 { Some(&views) } else { None }, + &mut self.object_mut(self.bundle)?.bytes_mut()[offset..offset + m::MAIN_CONFIG_SIZE]) + .map_err(|_| EINVAL)?; + let offset = index * 0x8000; + m::encode_root_for_profile(profile, role, &addresses, &instance, + &mut self.object_mut(self.roots)?.bytes_mut()[offset..offset + m::M5_ROOT_SIZE]) + .map_err(|_| EINVAL)?; + } + fence(Ordering::SeqCst); + dev_info!(dev.as_ref(), "M5 firmware graph ready: private {:#x}+{:#x}, shared {:#x}+{:#x}\n", + PRIVATE, private.total, BUNDLE, shared.total); + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/g17/initdata/outer_credit.rs b/drivers/gpu/drm/asahi/g17/initdata/outer_credit.rs new file mode 100644 index 0000000000000..08804ff1fc498 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/initdata/outer_credit.rs @@ -0,0 +1,130 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Pending capacity claims for M5's per-priority shared outer ring. Firmware +//! consumption frees posted slots; RAII cancellation frees unposted claims. + +use crate::g17::{availability::Availability, fw::channels::WORK_RING_SLOTS}; +use core::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering}; +use kernel::{prelude::*, sync::Arc}; + +const PRIORITIES: usize = 4; + +pub(super) struct Credits { + reserved: [AtomicU32; PRIORITIES], + published: [AtomicU64; PRIORITIES], + consumers: [AtomicU64; PRIORITIES], + scoped: AtomicBool, + available: Arc, +} + +pub(crate) struct OuterClaim { + domain: Arc, + priority: usize, + remaining: u32, +} + +/// Borrows the claim only. The caller must hold the device mutex from entry +/// through Drop: its temporarily lent credits are available to that publisher. +pub(crate) struct OuterScope<'a> { + claim: &'a mut OuterClaim, + before: u64, +} + +pub(super) fn free([a, b, producer]: [u32; 3]) -> Result { + if [a, b, producer].iter().any(|&v| v >= WORK_RING_SLOTS) { + return Err(EINVAL); + } + Ok(((a + WORK_RING_SLOTS - producer - 1) % WORK_RING_SLOTS) + .min((b + WORK_RING_SLOTS - producer - 1) % WORK_RING_SLOTS)) +} + +impl Credits { + pub(super) fn new() -> Result> { + Ok(Arc::new(Self { + reserved: [const { AtomicU32::new(0) }; PRIORITIES], + published: [const { AtomicU64::new(0) }; PRIORITIES], + consumers: [const { AtomicU64::new(u64::MAX) }; PRIORITIES], + scoped: AtomicBool::new(false), + available: Availability::new()?, + }, GFP_KERNEL)?) + } + + pub(super) fn available(&self) -> Arc { self.available.clone() } + + pub(super) fn waiting(&self) -> bool { self.available.has_waiters() } + + pub(super) fn capacity(&self, priority: u8, cursors: [u32; 3]) -> Result { + let pending = self.reserved.get(usize::from(priority)).ok_or(EINVAL)? + .load(Ordering::Acquire); + Ok(free(cursors)?.saturating_sub(pending)) + } + + /// Reserve under the sole publisher mutex. Off-lock cancellation may only + /// reduce reserved counts, so the sampled capacity is conservative. + pub(super) fn reserve(self: &Arc, priority: u8, count: u32, + cursors: [u32; 3]) -> Result + { + if count == 0 || count >= WORK_RING_SLOTS { return Err(EINVAL); } + if self.scoped.load(Ordering::Acquire) { return Err(EBUSY); } + let index = usize::from(priority); + let counter = self.reserved.get(index).ok_or(EINVAL)?; + let seen = (u64::from(cursors[0]) << 32) | u64::from(cursors[1]); + self.consumers[index].compare_exchange(u64::MAX, seen, Ordering::AcqRel, Ordering::Acquire).ok(); + if self.capacity(priority, cursors)? < count { return Err(EBUSY); } + counter.fetch_add(count, Ordering::AcqRel); + Ok(OuterClaim { domain: self.clone(), priority: index, remaining: count }) + } + + pub(super) fn scope<'a>(self: &Arc, claim: &'a mut OuterClaim) -> Result> { + if !Arc::ptr_eq(self, &claim.domain) { return Err(EINVAL); } + if self.scoped.swap(true, Ordering::AcqRel) { return Err(EBUSY); } + self.reserved[claim.priority].fetch_sub(claim.remaining, Ordering::AcqRel); + let before = self.published[claim.priority].load(Ordering::Acquire); + Ok(OuterScope { claim, before }) + } + + pub(super) fn counter(&self, priority: u8) -> Result<&AtomicU64> { + self.published.get(usize::from(priority)).ok_or(EINVAL) + } + + pub(super) fn progressed(&self, priority: u8, cursors: [u32; 3]) -> Result { + free(cursors)?; + let counter = self.consumers.get(usize::from(priority)).ok_or(EINVAL)?; + let current = (u64::from(cursors[0]) << 32) | u64::from(cursors[1]); + let previous = counter.swap(current, Ordering::AcqRel); + Ok(previous != u64::MAX && previous != current) + } +} + +impl Drop for OuterScope<'_> { + fn drop(&mut self) { + let domain = &self.claim.domain; + let published = domain.published[self.claim.priority].load(Ordering::Acquire) + .wrapping_sub(self.before); + // Every publication also checks unreserved physical capacity. If an + // enclosing transaction posts extra records, only this claim's own + // credits are consumed here; another claim can never be debited. + self.claim.remaining -= published.min(u64::from(self.claim.remaining)) as u32; + domain.reserved[self.claim.priority].fetch_add(self.claim.remaining, Ordering::AcqRel); + domain.scoped.store(false, Ordering::Release); + } +} + +impl OuterClaim { + /// Credits not yet consumed by a publication. A claim with none left can be dropped + /// anywhere: only returned credits notify waiters. + pub(crate) fn remaining(&self) -> u32 { + self.remaining + } +} + +impl Drop for OuterClaim { + /// As for physical queue claims, drop outside the device mutex because + /// availability notification can run DMA-fence callbacks. + fn drop(&mut self) { + if self.remaining != 0 { + self.domain.reserved[self.priority].fetch_sub(self.remaining, Ordering::AcqRel); + self.domain.available.notify(); + } + } +} diff --git a/drivers/gpu/drm/asahi/g17/initdata/v25g83.rs b/drivers/gpu/drm/asahi/g17/initdata/v25g83.rs index f7b1952685761..c0485c4932e48 100644 --- a/drivers/gpu/drm/asahi/g17/initdata/v25g83.rs +++ b/drivers/gpu/drm/asahi/g17/initdata/v25g83.rs @@ -130,7 +130,7 @@ impl InitData { calibration.max_power_mw, &calibration.leakage, ), - (None, Some(chip)) => (cfg.perf, chip.max_power_mw, &chip.leakage), + (None, Some(chip)) => (cfg.perf()?, chip.max_power_mw, &chip.leakage), _ => return Err(EINVAL), }; let hw_args = abi::HwDataArgs { @@ -165,7 +165,7 @@ impl InitData { })?; self.object_mut(self.region_c)? .initialize::(0, |region| { - region.init(cfg.region_c); + region.init(cfg.region_c()?); region.max_power_mw = max_power_mw; Ok(()) })?; diff --git a/drivers/gpu/drm/asahi/g17/initdata/work_pair.rs b/drivers/gpu/drm/asahi/g17/initdata/work_pair.rs new file mode 100644 index 0000000000000..eafd6daac4a85 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/initdata/work_pair.rs @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Two render queues sharing one outer ring. Capacity and both destinations +//! are checked before the host-port transaction can make either queue runnable. + +use crate::g17::fw::channels::{WorkSlot, WORK_RING_SLOTS}; +use core::sync::atomic::{fence, AtomicU32, AtomicU64, Ordering}; +use kernel::prelude::*; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) struct Window { + pub(super) start: u32, + pub(super) next: [u32; 2], +} + +impl Window { + pub(super) fn new([consumer, consumer_b, start]: [u32; 3]) -> Result { + if [consumer, consumer_b, start].iter().any(|&p| p >= WORK_RING_SLOTS) { + return Err(EINVAL); + } + let next = [(start + 1) % WORK_RING_SLOTS, (start + 2) % WORK_RING_SLOTS]; + if next.iter().any(|&p| p == consumer || p == consumer_b) { + return Err(EBUSY); + } + Ok(Self { start, next }) + } +} + +pub(in crate::g17) struct Publication<'a> { + published: Option<&'a AtomicU64>, + records: [WorkSlot; 2], + destinations: [*mut u8; 2], + producer: &'a AtomicU32, + window: Window, +} + +impl<'a> Publication<'a> { + /// SAFETY: Both destinations must identify distinct, writable WorkSlot + /// storage in the retained ring at window.start and window.next[0]. The + /// caller holds the sole host publication lock until this value is dropped. + pub(super) unsafe fn new( + records: [WorkSlot; 2], + destinations: [*mut u8; 2], + producer: &'a AtomicU32, + window: Window, + published: Option<&'a AtomicU64>, + ) -> Self { + Self { records, destinations, producer, window, published } + } + + pub(in crate::g17) fn next(&self) -> [u32; 2] { self.window.next } + + pub(in crate::g17) fn prepare(&self) { + for (record, destination) in self.records.iter().zip(self.destinations) { + // SAFETY: Constructor checked both live destinations. Records are + // initialized, padding-free integer structures owned by this value. + unsafe { + core::ptr::copy_nonoverlapping( + (record as *const WorkSlot).cast::(), destination, size_of::(), + ); + } + } + } + + /// Publish both records with one cursor update, after both inner queues and + /// their accounting are visible. No fallible operation follows the port write. + pub(in crate::g17) fn publish(self) -> [u32; 2] { + self.prepare(); + fence(Ordering::SeqCst); + self.producer.store(self.window.next[1], Ordering::Relaxed); + if let Some(published) = self.published { published.fetch_add(2, Ordering::Release); } + fence(Ordering::SeqCst); + self.window.next + } +} diff --git a/drivers/gpu/drm/asahi/g17/job.rs b/drivers/gpu/drm/asahi/g17/job.rs index fcd3a1a7a74f3..d5c07c1c2220e 100644 --- a/drivers/gpu/drm/asahi/g17/job.rs +++ b/drivers/gpu/drm/asahi/g17/job.rs @@ -7,6 +7,9 @@ use crate::driver::AsahiDrmDevice; use core::sync::atomic::{AtomicBool, AtomicI32, AtomicU8, Ordering}; use kernel::{ bindings, c_str, + cpu::CpuId, + cpumask::Cpumask, + workqueue, dma_fence::{Fence, RawDmaFence}, drm::sched, prelude::*, @@ -14,8 +17,9 @@ use kernel::{ }; pub(super) const TIMEOUT_MS: kernel::time::Msecs = super::timeout::POLL_MS; -/// One logical render queue owns one physical pair, whose active ring holds -/// `RENDER_DEPTH` commands. A deeper scheduler would only park run_job in +/// Each physical pair of a logical render queue holds `RENDER_DEPTH` commands +/// in its active ring, so a queue gets that many credits per pair it may hold +/// ([`Backend::render_pairs`]). A deeper scheduler would only park run_job in /// admission until a retirement; the scheduler waits for the credit instead. const RENDER_CREDITS: u32 = crate::hw::t8140::queues::RENDER_DEPTH as u32; const COMPUTE_CREDITS: u32 = crate::hw::t8140::queues::COMPUTE_DEPTH as u32; @@ -47,6 +51,9 @@ impl Engine { pub(crate) struct Order { pub(crate) sequence: u64, pub(crate) wait_through: [Option; 2], + /// What a render's tiling half waits on: `wait_through` unless the command + /// carries its own, never later, tiling prefixes. + pub(crate) tiling_wait_through: [Option; 2], } impl Order { @@ -87,9 +94,22 @@ impl Order { Ok(Self { sequence: sequences[engine.index()], wait_through, + tiling_wait_through: wait_through, }) } + /// Give the tiling half the narrower prefixes of `tiling`, which resolves the + /// command's tail against the same ioctl position. + pub(crate) fn narrow_tiling(&mut self, tiling: &Self) -> Result { + for index in 0..2 { + if tiling.wait_through[index].unwrap_or(0) > self.wait_through[index].unwrap_or(0) { + return Err(EINVAL); + } + } + self.tiling_wait_through = tiling.wait_through; + Ok(()) + } + pub(crate) fn contains(prefix: Option, sequence: u64) -> bool { sequence != 0 && prefix.is_some_and(|last| sequence <= last) } @@ -121,8 +141,8 @@ impl Packet { firmware: KVec, ) -> Result> { let footprint = match &command { - Validated::Render { attachments, .. } => { - context.vm().render_footprint(&attachments[1])? + Validated::Render { attachments, pass, .. } => { + context.vm().render_footprint(&attachments[1], pass.whole_attachments)? } Validated::Compute { .. } => None, }; @@ -191,10 +211,19 @@ impl Packet { } check_status(fence)?; } + self.check_firmware_producers()?; + self.completion.check_source_inputs()?; + Ok(()) + } + + /// Producers this packet waits for in firmware (another engine's commands that + /// were published, not complete, when it was scheduled). Checked before + /// publication, and again at retirement: a producer that fails after this + /// packet was published fails it too, although the firmware ran it. + pub(crate) fn check_firmware_producers(&self) -> Result { for fence in &self.firmware { check_status(fence)?; } - self.completion.check_source_inputs()?; Ok(()) } @@ -281,9 +310,32 @@ pub(crate) trait Backend: Send + Sync { None } + /// The device accepts command submission (an M5 GPU before its queues take submission + /// does not). + fn supports_submission(&self) -> bool { true } + /// Whether this backend's scheduler workers run in their queue creator's cluster (M5). + fn creator_cluster_scheduling(&self) -> bool { false } + /// Whether to keep compute input completion waits and wait on the host for other-engine + /// prefix producers: without the G17D forms an M5 kick may start while its ring + /// predecessor still runs. + fn host_producer_waits(&self) -> bool { false } + /// Whether a compute command may be waited for in firmware only when it declares that its + /// writes are visible at completion (DRM_ASAHI_COMPUTE_VISIBLE_COMPLETION). + fn firmware_sources_need_visible_completion(&self) -> bool { false } + /// The reclaim-safe queue for the scheduler's timeout work, whose handler signals the + /// fences of timed-out jobs. + fn fence_queue(&self) -> Arc; + /// Render pairs this logical queue may hold, fixed when it is created: its + /// render scheduler's credits are fixed then too. + fn render_pairs(&self) -> u32 { 1 } + fn supports_engine(&self, _engine: Engine) -> bool { true } + /// Device policy applied to a validated command before it is ordered (the M5 resource + /// plan). + fn prepare_command(&self, _command: &mut super::command::Validated) -> Result { Ok(()) } + /// Validate compute sizing without reserving a physical channel. Queued /// commands may wait on a host signal submitted after this ioctl returns. - fn ensure_compute(&self, scs_pages: u16) -> Result; + fn ensure_compute(&self, command: &super::command::Validated) -> Result; fn render_scs_blocks(&self, pages: [u16; 2]) -> Result { if pages != [0; 2] { return Err(EINVAL); } Ok(super::fw::freelist::RENDER_BLOCKS) @@ -302,6 +354,9 @@ pub(crate) trait Backend: Send + Sync { fn defer_timeout(&self, packet: &Arc, renewals: u8) -> bool; /// A long job of another queue is running beside `packet`. fn long_job_beside(&self, _packet: &Arc) -> bool { false } + /// A published job's deadline may be renewed while other work retires + /// ([`super::timeout::renew_shared`]). + fn renews_shared(&self) -> bool { true } /// Return true only after taking publication ownership and recording its result. fn try_early(&self, packet: &Arc) -> bool; fn count_backlog(&self, packet: &Packet); @@ -357,8 +412,9 @@ impl sched::JobImpl for Job { } // Other work retiring means the GPU is shared, not hung. So does another long job // running beside a long job, though neither retires until it ends. - let shared = super::timeout::retiring(super::recovery::since_retired(super::now_ns())) - || (job.packet.command.long_job() && job.backend.long_job_beside(&job.packet)); + let shared = job.backend.renews_shared() + && (super::timeout::retiring(super::recovery::since_retired(super::now_ns())) + || (job.packet.command.long_job() && job.backend.long_job_beside(&job.packet))); if super::timeout::renew_shared(job.packet.is_published(), shared, &mut job.shared_renewals) { return true; } @@ -432,10 +488,26 @@ impl Fences { impl Scheduler { pub(crate) fn new(dev: &AsahiDrmDevice, backend: Arc, engine: Engine) -> Result { let (credits, name) = match engine { - Engine::Render => (RENDER_CREDITS, c_str!("aurora_g17_render")), + Engine::Render => ( + RENDER_CREDITS.checked_mul(backend.render_pairs()).ok_or(EOVERFLOW)?, + c_str!("aurora_g17_render"), + ), Engine::Compute => (COMPUTE_CREDITS, c_str!("aurora_g17_compute")), }; - let scheduler = sched::Scheduler::new(dev.as_ref().as_ref(), 1, credits, 0, TIMEOUT_MS, name)?; + let device = dev.as_ref().as_ref(); + let submit = if backend.creator_cluster_scheduling() { + // The run and free work stays in the creating CPU's cluster, as the event work + // follows the submitter, instead of wherever an unbound worker last ran. + let queue = workqueue::OwnedQueue::new_ordered_reclaim(name)?; + queue.restrict(Cpumask::cluster_of(CpuId::current()))?; + Some(queue) + } else { + // drm_sched allocates its own ordered reclaim-safe queue. + None + }; + let scheduler = sched::Scheduler::new_with_queues( + device, 1, credits, 0, TIMEOUT_MS, name, submit, Some(backend.fence_queue()), + )?; let entity = sched::Entity::new(&scheduler, sched::Priority::Kernel)?; Ok(Self { entity, diff --git a/drivers/gpu/drm/asahi/g17/kick.rs b/drivers/gpu/drm/asahi/g17/kick.rs index 69beeac76ed95..5f49bf1b5093d 100644 --- a/drivers/gpu/drm/asahi/g17/kick.rs +++ b/drivers/gpu/drm/asahi/g17/kick.rs @@ -20,7 +20,7 @@ use super::{ fw::{kick::*, queue::DataMaster}, object::{Allocator, CpuMap, KernelObject}, }; -use crate::{hw::t8140, mem, mmu}; +use crate::{mem, mmu}; pub(super) const QID_COUNT: usize = QID_MAX as usize + 1; @@ -56,7 +56,9 @@ const _: () = { assert!(queue_limit(0) == crate::mmu::MAX_EXECUTION_CONTEXTS); // Compute keeps this many channels even with every execution context open. assert!(QID_COUNT - FULL_RENDER_RESERVE >= 4); - assert!(crate::hw::t8140::queues::RENDER_PAIRS_PER_OWNER == 1); + // The reserve counts one pair per owner. A second pair of an owner comes + // only from surplus (g17::pairs), never from this reserve. + assert!(FULL_RENDER_RESERVE == 2 * crate::mmu::MAX_EXECUTION_CONTEXTS as usize + 2); }; /// Stable physical-channel identity. The owner must not be reused within a @@ -398,11 +400,27 @@ enum Backing { Render(RenderRing), } +/// A client VM's view of a kick ring. The host writes the entries through the kernel's mapping +/// and the firmware reads them through its own; a client's GPU work only reads them, so its +/// view is read-only: a client cannot rewrite an entry after the kernel built and checked it. +const CLIENT_KICK_PROT: mmu::Prot = mmu::PROT_GPU_SHARED_RO; + /// A render ring's pages and the shared lower reservation clients alias. struct RenderStorage { // The lower alias drops before its backing. low: mmu::KernelMapping, high: KernelObject, + /// The client VM the storage was created for, whose alias of it lasts as long as that VM. + vm: u64, + /// That VM's status, which tells when its client closed it. + status: Option>, +} + +impl RenderStorage { + /// The client closed the VM this storage was created for. + fn closed(&self) -> bool { + self.status.as_ref().is_some_and(|status| status.client_closed()) + } } /// Render ring storage outlives its rings for the life of the device. @@ -442,12 +460,27 @@ impl RenderRings { ) } - fn take(&self) -> Option { + /// Reusable storage created for `client`, else storage whose VM its client has closed, + /// which then belongs to `client`. Storage of another open VM is never handed out: that VM + /// keeps its read-write alias of the pages, so it could read and write the new ring. + fn take(&self, client: &mmu::Vm) -> Option { + let vm = client.id(); let mut stock = self.stock.lock(); - let index = stock.retired.iter().rposition(|(reusable, _)| *reusable)?; + let reusable = |own: bool| { + stock.retired.iter().rposition(|(reusable, storage)| { + *reusable && if own { storage.vm == vm } else { storage.closed() } + }) + }; + let index = reusable(true).or_else(|| reusable(false))?; let last = stock.retired.len() - 1; stock.retired.swap(index, last); - stock.retired.pop().map(|(_, storage)| storage) + let (_, mut storage) = stock.retired.pop()?; + drop(stock); + if storage.vm != vm { + storage.vm = vm; + storage.status = client.status().cloned(); + } + Some(storage) } /// Reserves the return of one more, just created storage. @@ -538,10 +571,10 @@ impl Queue { mmu::PROT_GPU_SHARED_RW, CpuMap::WriteCombined, )?; - let range = t8140::dynamic::upper(alloc.uat.geometry().kernel_range()); +// The UAT reserves the boot graph and firmware register aperture. let high = low.alias_in( alloc.uat.kernel_vm(), - range, + alloc.uat.kernel_dynamic_range(), mmu::UAT_PGSZ as u64, mmu::PROT_FW_SHARED_RW, )?; @@ -566,7 +599,7 @@ impl Queue { if id.engine == DataMaster::Compute { return Err(EINVAL); } - if let Some(storage) = rings.take() { + if let Some(storage) = rings.take(client) { let mut ring = RenderRing { storage: ManuallyDrop::new(storage), rings: rings.clone(), @@ -579,7 +612,7 @@ impl Queue { let alias = ring.storage .high - .prepare_alias(client, address, mmu::PROT_GPU_SHARED_RW, true); + .prepare_alias(client, address, CLIENT_KICK_PROT, false); // On failure the storage returns to the stock and a fresh ring is built. if let Ok(alias) = alias { return Ok(( @@ -593,13 +626,22 @@ impl Queue { )); } } + let (vm, status) = (client.id(), client.status().cloned()); + // The firmware's mapping sits in the upper half, which every execution context includes. + // On M5 it is firmware-only, as a compute ring's is; the GPU reads the ring at its lower + // address. + let firmware_prot = if alloc.uat.is_m5() { + mmu::PROT_FW_SHARED_RW + } else { + mmu::PROT_GPU_FW_SHARED_RW + }; let high = alloc.kernel( KICK_RING_SIZE, mmu::UAT_PGSZ as u64, - mmu::PROT_GPU_FW_SHARED_RW, + firmware_prot, CpuMap::WriteCombined, )?; - let mut range = t8140::dynamic::LOWER; + let mut range = alloc.uat.kernel_lower_dynamic_range(); let (low, client) = loop { let low = high.alias_in( alloc.uat.kernel_lower_vm(), @@ -611,7 +653,7 @@ impl Queue { .iova() .checked_add(KICK_RING_SIZE as u64) .ok_or(EOVERFLOW)?; - match high.map_alias(client, low.iova(), mmu::PROT_GPU_SHARED_RW) { + match high.map_alias(client, low.iova(), CLIENT_KICK_PROT) { Ok(client) => break (low, client), Err(error) if error == ENOSPC && end < range.end => { drop(low); @@ -620,7 +662,7 @@ impl Queue { Err(error) => return Err(error), } }; - let storage = RenderStorage { low, high }; + let storage = RenderStorage { low, high, vm, status }; // Count the storage only once it exists. Without room to return it, // nothing has aliased it but `client`, which drops first. if let Err(error) = rings.admit() { @@ -667,7 +709,31 @@ impl Queue { pub(crate) fn map_client(&mut self, vm: &mmu::Vm) -> Result { let address = self.low_va(); self.object() - .map_alias(vm, address, mmu::PROT_GPU_SHARED_RW) + .map_alias(vm, address, CLIENT_KICK_PROT) + } + + /// Clears an idle compute ring before another VM is given a view of it, so the new owner + /// cannot read the entries the previous owner's commands left. The firmware has consumed + /// every entry; the host's position in the ring is kept. + pub(crate) fn clear_ring(&mut self) -> Result { + let Backing::Compute { low, .. } = &mut self.backing else { + return Err(EINVAL); + }; + low.bytes_mut().fill(0); + fence(Ordering::SeqCst); + Ok(()) + } + + /// Clears an idle render ring before a context of another VM is given a view of it, so the + /// new owner cannot read the entries earlier owners' commands left. The firmware has + /// consumed every entry; the host's position in the ring is kept. + pub(crate) fn clear_render_ring(&mut self) -> Result { + let Backing::Render(ring) = &mut self.backing else { + return Err(EINVAL); + }; + ring.storage.high.bytes_mut().fill(0); + fence(Ordering::SeqCst); + Ok(()) } /// Reuse a permanent client view only when every page names this ring. @@ -676,14 +742,25 @@ impl Queue { Backing::Compute { low, .. } => low, Backing::Render(ring) => &ring.storage.high, }; - object.prepare_alias(vm, self.low_va(), mmu::PROT_GPU_SHARED_RW, true) + object.prepare_alias(vm, self.low_va(), CLIENT_KICK_PROT, false) } /// Both aliases are owned mappings of the same full ring. Publish their /// page-table updates to each live translation regime before installation. + /// + /// With published roots each alias reaches every context whose lower or + /// upper half names its VM's root: the low alias lives in the kernel lower + /// VM, the firmware alias in the kernel upper VM, whose root sits in the + /// upper half of every execution context. Under the handoff the first four + /// contexts are invalidated. pub(crate) fn prepare_compute_install(&self) -> Result { - if self.id.engine != DataMaster::Compute { + let Backing::Compute { high, low } = &self.backing else { return Err(EINVAL); + }; + if high.tracks_contexts() { + low.publish_to_contexts()?; + high.publish_to_contexts(); + return Ok(()); } mem::sync(); for asid in 0..4 { @@ -709,10 +786,17 @@ impl Queue { _ => self.timestamp.prev(), }) } + /// The parent's predecessor, while this queue remembers a previous kick: a G17D fragment + /// kick names it after its pipelined self-parent. Never on the first kick's synthetic + /// parent; recovery clears the history, and with it this. + pub(crate) fn ancestor(&self) -> Option { + self.previous.map(KickTimestamp::prev) + } pub(crate) fn prepare_entry(&mut self, args: &KickArgs<'_>) -> Result> { if args.qid != self.id.qid || args.timestamp != self.timestamp || args.parent != self.parent() + || args.ancestor.is_some_and(|ancestor| Some(ancestor) != self.ancestor()) { return Err(EINVAL); } diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 32f04b38372c3..4e68d0bd74a47 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -21,12 +21,15 @@ //! nests inside nothing. Capacity waits block on an availability domain signalled by the //! release that provides the capacity, never on a sleep. -mod availability; +pub(crate) mod availability; mod buffer; mod calibration; mod channel; mod command; pub(crate) use command::Attachments; +// M5 render encodings, reached only once the M5 render runtime is wired. +#[allow(dead_code)] +mod render_resources; mod completion; pub(crate) mod context; mod data_segment; @@ -43,6 +46,7 @@ mod fence; mod freelist; pub(crate) mod fw; pub(crate) mod hazard; +mod pairs; mod ids; mod initdata; mod job; @@ -61,6 +65,7 @@ pub(crate) mod stats; pub(crate) mod status; mod teardown; pub(crate) mod telemetry; +mod thermal; mod timeout; mod validation; @@ -157,6 +162,41 @@ pub(crate) struct FixedBuffer { pub(crate) size: usize, } +/// Firmware protocol of a G17 GPU. +#[derive(Copy, Clone, PartialEq, Eq)] +pub(crate) enum FirmwareProfile { + /// The firmware publishes its kernel roots and the driver owns the page tables (A18 Pro, + /// M4 Pro). + PublishedRoots, + /// The M5 firmware on bare metal: both roles publish their kernel roots + /// ([`mmu::UatFirmware::PublishedRoots`]) and the driver owns the page tables. + M5PublishedRoots, +} + +impl Config { + /// The fixed parameter-buffer metrics buffer of a 26A428 or 25G83 graph. + pub(crate) fn pm_metrics(&self) -> Result<&FixedBuffer> { + self.pm_metrics.as_ref().ok_or(EINVAL) + } + + /// The performance tables of a 26A428 or 25G83 graph. + pub(crate) fn perf(&self) -> Result<&'static fw::initdata::PerfConfig> { + self.perf.ok_or(EINVAL) + } + + /// Region C of a 26A428 or 25G83 graph. + pub(crate) fn region_c(&self) -> Result<&'static fw::initdata::RegionCConfig> { + self.region_c.ok_or(EINVAL) + } +} + +impl FirmwareProfile { + /// The M5 firmware's protocol (26A428 InitData graph, queues and submission). + pub(crate) const fn is_m5(self) -> bool { + matches!(self, Self::M5PublishedRoots) + } +} + /// Static configuration of a SoC with a G17 GPU. pub(crate) struct Config { /// SoC identifier. @@ -172,14 +212,18 @@ pub(crate) struct Config { pub(crate) secondary_root_offset: Option, /// Where the coprocessors' crash log buffers live. pub(crate) crash_buffers: CrashBuffers, - /// Parameter buffer page metrics shared with the firmware. - pub(crate) pm_metrics: FixedBuffer, + /// Firmware protocol. + pub(crate) profile: FirmwareProfile, + /// Parameter buffer page metrics shared with the firmware; `None` where the firmware graph + /// places its own (M5). + pub(crate) pm_metrics: Option, /// Register windows the firmware maps through the hardware data. pub(crate) windows: &'static [fw::initdata::RegisterWindow], /// Register-map slots that are reserved but have no window. pub(crate) reserved_slots: &'static [usize], - /// Per-SoC parts of the performance tables. - pub(crate) perf: &'static fw::initdata::PerfConfig, + /// Per-SoC parts of the performance tables; `None` where the firmware graph has its own + /// (M5, [`fw::m5::initdata`]). + pub(crate) perf: Option<&'static fw::initdata::PerfConfig>, /// Number of operating points the device tree lists. pub(crate) opps: usize, /// UAT policy where it differs from the A18 Pro's. @@ -191,9 +235,11 @@ pub(crate) struct Config { /// Enable the GPU-to-fabric bridge and dynamic core power gating before the firmware /// starts (A18 Pro). The M4 Pro has only run with neither written. pub(crate) fabric_bridge: bool, - /// The SoC's values of region C. - pub(crate) region_c: &'static fw::initdata::RegionCConfig, + /// The SoC's values of region C; `None` where the firmware graph has its own (M5). + pub(crate) region_c: Option<&'static fw::initdata::RegionCConfig>, pub(crate) chip: Option<&'static fw::initdata::v25g83::Chip>, + /// The M5 board whose HW data and power policy the M5 graph is built with; `None` off M5. + pub(crate) m5: Option<&'static fw::m5::initdata::Board>, /// Physical address of the firmware's per-slot recovery keys, read while the firmware is /// halted to attribute a fault; `None` where it is unknown (recovery then relies on the /// firmware's own blame). @@ -645,6 +691,8 @@ struct Firmware { recovery: recovery::State, effort: power::Effort, hold: power::Hold, + /// M5: host GPU power budget governor (Region C's budget word). + governor: thermal::Governor, registration_power_release_pending: bool, variant: &'static fw::Variant, queues: runtime::Registry, @@ -658,6 +706,18 @@ struct Firmware { uat: Arc, } +/// Set once this module is pinned until reboot for a session that is never freed: an M5 +/// session once a role has run, which no engine retirement witness releases +/// ([`initdata::InitData::has_engine_retirement_witness`]). Its RTKit and mailbox callbacks +/// run this module's code. +static SESSION_RETAINED: AtomicBool = AtomicBool::new(false); + +/// Pins this module until reboot for a session that is never freed, once +/// ([`crate::retain::retain_module`]). ENODEV once the module's removal has begun. +fn retain_session() -> Result { + crate::retain::retain_module(&SESSION_RETAINED) +} + impl Drop for Firmware { fn drop(&mut self) { if let Some(secondary) = self.secondary.as_mut() { @@ -697,10 +757,16 @@ impl Firmware { /// Holds idle descent while a fresh physical render pair is registered. fn acquire_registration_power(&mut self) -> Result { + if !self.variant.registration_power_hold { + return Ok(()); + } power::acquire(self) } fn release_registration_power(&mut self) -> Result { + if !self.variant.registration_power_hold { + return Ok(()); + } // An idle hold keeps the cores powered past registration; it releases them itself. if self.hold.active() { self.registration_power_release_pending = false; @@ -709,13 +775,102 @@ impl Firmware { power::release(self) } + /// Retry a temporarily unavailable transaction once under a registration + /// power hold. Other access errors propagate to the retained owner. + fn submit_m5_work(&mut self, submission: &fw::m5::queue::HardwareSubmission) -> Result { + match self.bridge.regs.submit_m5_work(submission) { + Err(error) if error == EAGAIN => { + self.acquire_registration_power()?; + self.bridge.regs.submit_m5_work(submission) + } + result => result, + } + } + + /// Both outer slots and their capacity are checked before any host-port + /// write. An EAGAIN retry drops the old views and revalidates them under a + /// power hold. After a successful transaction, publication is infallible. + fn submit_m5_render_pair( + &mut self, + priority: u8, + records: &[fw::channels::WorkSlot; 2], + submission: &fw::m5::queue::HardwareSubmission, + commit_accounting: impl FnOnce() -> Result, + publish_inner: impl FnOnce(), + deferred: bool, + ) -> Result<[u32; 2]> { + let mut commit_accounting = Some(commit_accounting); + let mut publish_inner = Some(publish_inner); + for attempt in 0..2 { + let pair = self.init.m5_work_pair(priority, &records[0], &records[1])?; + pair.prepare(); + if let Some(commit) = commit_accounting.take() { commit()?; } + match self.bridge.regs.submit_m5_work(submission) { + Err(error) if error == EAGAIN && attempt == 0 => { + drop(pair); + self.acquire_registration_power()?; + } + Err(error) => return Err(error), + Ok(()) => { + if let Some(publish) = publish_inner.take() { publish(); } + return Ok(if deferred { pair.next() } else { pair.publish() }); + } + } + } + Err(EAGAIN) + } + /// Called at actual submission publication, not while a queued job is merely being built. fn note_submission(&mut self) -> Result { + self.primary.state.shared.note_submitter(); self.port.note_gpu_activity(); + self.primary.state.shared.note_gpu_activity(); self.effort.submit(); + self.effort.note_waiters(now_ns(), fence::waiters()); self.apply_effort(power::FULL_EFFORT) } + /// One governor step with the hotspot temperature reading; returns whether to run again and + /// what the CPU ceiling should do (applied by the caller without the device lock). + fn governor_tick(&mut self, temp_mc: Option, fans: bool) + -> (bool, thermal::CpuAdvice, bool) + { + let now = now_ns(); + // Work published since the last step, with or without the device mutex. + let published = self.primary.state.shared.governor_activity_ns.swap(0, Ordering::AcqRel); + if published != 0 { + self.governor.note_activity(published); + } + // Effort remains asserted until submitted work retires. A long job + // must not look idle merely because it has no subsequent submission. + if self.effort.applied() != 0 && thermal::requested() { + self.governor.note_activity(now); + } + let Ok(current) = self.init.m5_power_budget() else { + return (false, thermal::CpuAdvice::Release, false); + }; + self.governor.set_maximum(current); + let power = self.init.m5_gpu_power_mw().unwrap_or(0); + let clamped = self.init.m5_firmware_clamped().unwrap_or(false); + let (publish, again) = self.governor.tick(temp_mc, power, clamped, now); + if let Some(budget) = publish { + if self.init.publish_m5_power_budget(budget).is_err() { + return (false, thermal::CpuAdvice::Release, false); + } + } + let advice = self.governor.cpu_advice(power, now); + let want_fans = self.governor.fans_wanted(now); + // Keep ticking while the fans are still boosted, so the hold can expire. + (again || fans, advice, want_fans) + } + + /// Restore the calibrated maximum budget (shutdown). + fn governor_restore(&mut self) { + if let Some(budget) = self.governor.restore() { + let _ = self.init.publish_m5_power_budget(budget); + } + } + fn apply_effort(&mut self, effort: u32) -> Result { Self::apply_shared_effort(&self.init, &mut self.effort, &self.port, effort) } @@ -726,13 +881,16 @@ impl Firmware { port: &port::Port, effort: u32, ) -> Result { - if state.applied() == effort && init.effort()? == effort { + // The performance-controller demand follows the requested effort; the firmware is + // given, and the state records, the M5 value (InitData::submission_effort). + let published = init.submission_effort(effort, state.waited()); + port.set_effort_full(effort == power::FULL_EFFORT); + if state.applied() == published && init.effort()? == published { return Ok(()); } - init.publish_effort(effort)?; - trace::message(trace::EFFORT, u64::from(effort)); - state.did_apply(effort); - port.set_effort_full(effort == power::FULL_EFFORT); + init.publish_effort(published)?; + trace::message(trace::EFFORT, u64::from(published)); + state.did_apply(published); Ok(()) } @@ -758,6 +916,10 @@ impl Firmware { if submitted { self.effort.submit(); } + // A waiter that began or ended since the last note moves the effort while work streams. + if self.effort.streaming() && self.effort.note_waiters(now_ns, fence::waiters()) { + self.apply_effort(power::FULL_EFFORT)?; + } let effort = match self.effort.idle(now_ns, in_flight || submitted) { power::IdleAction::None => None, power::IdleAction::Clear => { @@ -784,6 +946,12 @@ impl Firmware { /// refused control (recovery in progress) is retried on a later check. fn idle_hold(&mut self, now_ns: u64, in_flight: bool) -> Option { const RETRY_NS: u64 = 1_000_000; + // The M5 firmware powers the cores for work by its own policy, and under a host + // hold J775D's core-power witness reads unpowered until the hold is released. M5 + // takes no host hold, whatever delay the tree or sysfs asks for. + if self.init.is_m5() { + return None; + } let hold_ms = power::IDLE_HOLD_MS.load(Ordering::Relaxed); for _ in 0..2 { match self @@ -846,6 +1014,28 @@ impl power::Control for Firmware { } } +/// Every feature the M5 command set reports. GET_PARAMS leaves out the MSAA compression +/// and mesh storage bits while those paths are off. +const M5_FEATURES: u64 = (uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_COMPUTE_RESOURCES + | uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_COMPUTE_NO_QUERY_ACCESS + | uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_RENDER_RESOURCES + | uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_RENDER_RESOURCES_V2) as u64 + | crate::file::DRM_ASAHI_FEATURE_SPARSE_TILES + | crate::file::DRM_ASAHI_FEATURE_COMPUTE_MATRIX + | crate::file::DRM_ASAHI_FEATURE_RENDER_SHADER_WRITES + | crate::file::DRM_ASAHI_FEATURE_RENDER_STAGE_PREFIXES_V2 + | crate::file::DRM_ASAHI_FEATURE_WHOLE_ATTACHMENT_RENDERS + | M5_MSAA_COMPRESSION_FEATURES + | crate::file::DRM_ASAHI_FEATURE_TASK_MESH_STORAGE; +const M5_MSAA_COMPRESSION_FEATURES: u64 = crate::file::DRM_ASAHI_FEATURE_M5_DEPTH_MSAA_COMPRESSION + | crate::file::DRM_ASAHI_FEATURE_M5_STENCIL_MSAA_COMPRESSION; +/// Features 24 and 25, which no kernel may report again (see the UAPI header): userspace +/// that knows them sends staged renders whose prefix fields the render command gives to +/// its ray-query stages. +const RETIRED_FEATURES: u64 = (uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_COMPUTE_STAGE_BARRIERS + | uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_RENDER_STAGE_PREFIXES) as u64; +const _: () = assert!(M5_FEATURES & RETIRED_FEATURES == 0); + /// Event work item ID. const EVENT_WORK: u64 = 0; /// Short control/retirement polls do not wait behind the longer idle deadline. @@ -854,6 +1044,7 @@ const IDLE_WORK: u64 = 2; const GROW_WORK: u64 = 3; const RECLAIM_WORK: u64 = 4; const FEED_WORK: u64 = 5; +const GOVERNOR_WORK: u64 = 6; /// Device state shared with the coprocessor callbacks and the event worker. #[pin_data] @@ -869,10 +1060,18 @@ struct Shared { variant: &'static fw::Variant, /// Layout of compute queues' operand-state buffers for this GPU's cores. operand_state: &'static fw::compute::OperandState, + /// The M5 board ([`Config::m5`]) of an M5 profile. + m5_board: Option<&'static fw::m5::initdata::Board>, /// Layout of renders' deflake buffers for this GPU's cores. deflake: fw::render::Deflake, compute_blocks: usize, descriptor_flags: [u32; 2], + /// Firmware events and work run: off for an M5 profile without the bare-metal boot. + submission_enabled: bool, + /// Such an M5 profile admits no render command. + firmware_test: bool, + /// The M5 shared-queue resource policy (frozen device topology). + render_policy: Option, crashed: AtomicBool, feed: Arc, #[pin] @@ -904,6 +1103,25 @@ struct Shared { poll: DelayedWork, #[pin] idle: DelayedWork, + /// M5: the thermal governor runs (the firmware has the host power budget word). + governs: bool, + #[pin] + governor: DelayedWork, + governor_armed: AtomicBool, + /// When work was last published while the governor is requested (0: none since its last + /// step). Publishers without the device mutex set it too. + governor_activity_ns: AtomicU64, + /// The queue of all work that signals job fences: the event work, and every scheduler's + /// timeout work. Memory reclaim may wait on those fences, so it is reclaim-safe. On M5 it is + /// unbound, so the event work runs in the affinity pod (the L2) of the CPU that last + /// published work, not on whichever CPU the completion IRQ thread first ran on; on a two-die + /// part that keeps completion processing on the submitter's die. Elsewhere it is per-CPU and + /// high priority, as `system_highpri_wq`. + events: Arc, + /// M5: the event work and the scheduler workers follow the submitting CPU. + follows_submitter: bool, + /// M5: the CPU of the last publication, `u32::MAX` before the first. + submitter: AtomicU32, #[pin] grow: Work, #[pin] @@ -918,6 +1136,10 @@ struct Shared { /// The device lock. `None` before boot completes and after detachment. #[pin] state: Mutex>>, + /// M5: the CPU frequency ceiling the thermal governor shares the SoC's heat with, and the + /// fan boost. Never taken with `state` held. + #[pin] + actuators: Mutex, #[pin] changed: CondVar, /// The port of the session in `state`; null before boot and once detachment begins. @@ -934,6 +1156,30 @@ impl_has_delayed_work! { impl_has_delayed_work! { impl HasDelayedWork for Shared { self.idle } } +impl_has_delayed_work! { + impl HasDelayedWork for Shared { self.governor } +} +impl WorkItem for Shared { + type Pointer = Arc; + fn run(this: Arc) { + this.governor_armed.store(false, Ordering::Release); + // Sensor reads may sleep: take them before the device lock. + let temp = thermal::hotspot_temperature_mc(); + let fans = this.actuators.lock().fans.boosted(); + let (again, advice, want_fans) = { + let mut state = this.state.lock(); + match (*state).as_deref_mut() { + Some(firmware) => firmware.governor_tick(temp, fans), + None => (false, thermal::CpuAdvice::Release, false), + } + }; + // Frequency QoS updates and SMC accesses may sleep: outside the device lock. + this.actuators.lock().apply(advice, want_fans, temp, now_ns()); + if again { + this.queue_governor(); + } + } +} impl_has_work! { impl HasWork for Shared { self.grow } } @@ -1042,8 +1288,35 @@ impl Shared { /// during a pass. The pass skips the housekeeping that nothing needs (see /// `runtime_event_turn`). Callable from any context. fn queue_event_records(self: &Arc) { + if !self.submission_enabled { + return; + } // Already queued if this fails, which is just as good. - let _ = workqueue::system_highpri().enqueue::, EVENT_WORK>(self.clone()); + let _ = match self.submitter() { + Some(cpu) => self.events.enqueue_on::, EVENT_WORK>(cpu, self.clone()), + None => self.events.enqueue::, EVENT_WORK>(self.clone()), + }; + } + + /// The queue for the schedulers' timeout work, which signals the fences of timed-out jobs. + pub(super) fn fence_queue(&self) -> Arc { + self.events.clone() + } + + /// Whether the event work and the scheduler workers follow the submitting CPU (M5). + pub(super) fn follows_submitter(&self) -> bool { + self.follows_submitter + } + + /// Records the publishing CPU for the event work's placement (M5). + fn note_submitter(&self) { + if self.follows_submitter { + self.submitter.store(kernel::cpu::CpuId::current().as_u32(), Ordering::Relaxed); + } + } + + fn submitter(&self) -> Option { + kernel::cpu::CpuId::from_u32(self.submitter.load(Ordering::Relaxed)) } fn queue_grow(self: &Arc) { @@ -1089,6 +1362,23 @@ impl Shared { .enqueue_delayed::, POLL_WORK>(self.clone(), msecs_to_jiffies(1)); } + fn queue_governor(self: &Arc) { + if !self.governor_armed.swap(true, Ordering::AcqRel) { + let _ = workqueue::system_unbound().enqueue_delayed::, GOVERNOR_WORK>( + self.clone(), msecs_to_jiffies(thermal::PERIOD_MS)); + } + } + + /// M5: work was published. Arms the thermal governor while it is requested. Called at + /// actual publication, by publishers with or without the device mutex; callable from any + /// context. + pub(super) fn note_gpu_activity(self: &Arc) { + if self.governs && thermal::requested() { + self.governor_activity_ns.store(now_ns().max(1), Ordering::Release); + self.queue_governor(); + } + } + fn queue_idle(self: &Arc, delay_ns: u64) { // Effort::idle bounds this by the 16 ms idle interval. Round up so a // timer never changes the firmware effort word before that deadline. @@ -1196,12 +1486,37 @@ impl Gpu { ); return Err(ENODEV); } + let m5 = cfg.profile.is_m5(); + if m5 { + let board = cfg.m5.ok_or(EINVAL)?; + // The M5 layouts are sized for every core slot of the board, fused-off cores + // included, so the GPU is matched on clusters and slots, not on enabled cores. + if id.num_clusters != board.clusters || id.num_clusters * id.num_cores != board.gpu_cores + { + dev_err!( + dev.as_ref(), + "M5 board expects {} clusters of {} core slots, the GPU reports {} x {}\n", + board.clusters, + board.gpu_cores / board.clusters, + id.num_clusters, + id.num_cores + ); + return Err(ENODEV); + } + fw::m5::pool::select_render_pool(board); + thermal::select_board(board); + pairs::select_board(board); + } let geometry = mmu::UatGeometry { ias: cfg.uat_ias, oas: cfg.uat_oas, }; - let max_frequency_khz = Self::max_frequency_khz(dev)?; + let max_frequency_khz = if m5 { + cfg.m5.ok_or(EINVAL)?.perf_states[13].0 * 1000 + } else { + Self::max_frequency_khz(dev)? + }; let node = dev.as_ref().as_ref().of_node().ok_or(EIO)?; let idle_delay = power::idle_delay(node.get_opt_property(c_str!("apple,idle-off-delay-ms"))?)?; @@ -1217,7 +1532,7 @@ impl Gpu { let preparations = preparation::Gate::with_waiters(Some(progress.clone()))?; let renders = availability::Availability::new()?; let reclaim = runtime::teardown::ReclaimBatch::new()?; - let feed = feed::Feed::new()?; + let feed = feed::Feed::new(m5)?; let alternate = cfg.core_layout.cores.is_some_and(|(clusters, cores)| { clusters == id.num_clusters && cores != active_cores @@ -1247,6 +1562,17 @@ impl Gpu { "Experimental GPU topology admission: command-buffer offsets are not yet validated\n" ); } + let render_policy = if m5 { Some(render_resources::M5Policy::new(&id)?) } else { None }; + // The bare-metal M5 boot runs both engines; other M5 profiles take no submission. + let submits = !m5 || cfg.profile == FirmwareProfile::M5PublishedRoots; + let events = Arc::new( + if m5 { + workqueue::OwnedQueue::new_unbound_reclaim(c"aurora_g17_events", true)? + } else { + workqueue::OwnedQueue::new_percpu_reclaim(c"aurora_g17_events", true)? + }, + GFP_KERNEL, + )?; let shared = Arc::pin_init( pin_init!(Shared { drm <- new_mutex!(Some(dev.into()), "g17::Shared::drm"), @@ -1255,9 +1581,13 @@ impl Gpu { active_cores, variant, operand_state: &cfg.core_layout.operand_state, + m5_board: cfg.m5.filter(|_| m5), deflake: cfg.core_layout.deflake, compute_blocks: cfg.core_layout.compute_blocks, descriptor_flags: [id.perf_control, id.perf_map[0]], + submission_enabled: submits, + firmware_test: !submits, + render_policy, crashed: AtomicBool::new(false), feed, feed_work <- new_work!("g17::Shared::feed_work"), @@ -1272,11 +1602,19 @@ impl Gpu { housekeeping_ns: AtomicU64::new(0), poll <- new_delayed_work!("g17::Shared::poll"), idle <- new_delayed_work!("g17::Shared::idle"), + governs: m5, + governor <- new_delayed_work!("g17::Shared::governor"), + governor_armed: AtomicBool::new(false), + governor_activity_ns: AtomicU64::new(0), + events, + follows_submitter: m5, + submitter: AtomicU32::new(u32::MAX), grow <- new_work!("g17::Shared::grow"), reclaim_work <- new_work!("g17::Shared::reclaim_work"), reclaim <- new_mutex!(reclaim, "g17::Shared::reclaim"), stop <- new_mutex!((), "g17::Shared::stop"), state <- new_mutex!(None, "g17::Shared::state"), + actuators <- new_mutex!(thermal::Actuators::new(), "g17::Shared::actuators"), changed <- new_condvar!("g17::Shared::changed"), port: core::sync::atomic::AtomicPtr::new(core::ptr::null_mut()), }), @@ -1318,7 +1656,7 @@ impl Gpu { }, GFP_KERNEL, )?; - let firmware = Self::boot(pdev, dev, cfg, regs, geometry, &gpu.shared)?; + let firmware = Self::boot(pdev, dev, cfg, regs, geometry, &gpu.id, &gpu.shared)?; gpu.shared .port .store(Arc::as_ptr(&firmware.port).cast_mut(), Ordering::Release); @@ -1421,6 +1759,175 @@ impl Gpu { Ok((freq_hz.ok_or(EINVAL)? / 1000).try_into()?) } + /// Starts the M5 firmware on bare metal: both roles boot and publish their kernel roots, + /// the M5 graph is built in the kernel window they share, then the M5 preparation, + /// InitData to the primary and then the secondary, POWER_ON_DONE to both, and both opening + /// records retired. + fn boot_m5( + pdev: &platform::Device>, + dev: &AsahiDrmDevice, + cfg: &'static Config, + regs: regs::G17Resources, + geometry: mmu::UatGeometry, + id: ®s::G17Id, + shared: &Arc, + ) -> Result> { + if cfg.profile != FirmwareProfile::M5PublishedRoots { + return Err(ENODEV); + } + let board = cfg.m5.ok_or(EINVAL)?; + // Names the step a failed bootstrap stopped at. + let at = |step: &'static str| { + move |e: &Error| { + dev_err!(dev.as_ref(), "M5 bare-metal bootstrap failed at {}: {:?}\n", step, e) + } + }; + let uat = Arc::new( + mmu::Uat::new( + dev.into(), + geometry, + true, + mmu::UatFirmware::PublishedRoots { + secondary_root_offset: cfg.secondary_root_offset, + }, + cfg.uat_quirks, + ) + .inspect_err(at("UAT"))? + .with_m5(), + GFP_KERNEL, + )?; + // The firmware samples the kernel lower root's top-level entries only when it starts: + // link the tables of every lower range the M5 graph and its queues use first. + uat.prepare_lower_tables(&initdata::InitData::m5_lower_links( + uat.kernel_lower_dynamic_range(), + )) + .inspect_err(at("lower links"))?; + let primary = Coprocessor::new(pdev, dev, cfg, &uat, shared, Role::Primary) + .inspect_err(|e| dev_err!(dev.as_ref(), "Primary coprocessor setup failed: {:?}\n", e))?; + let secondary = Coprocessor::new(pdev, dev, cfg, &uat, shared, Role::Secondary) + .inspect_err(|e| { + dev_err!(dev.as_ref(), "Secondary coprocessor setup failed: {:?}\n", e) + })?; + let render_global = uat.new_vm(0, geometry.user_range()).inspect_err(at("render VM"))?; + let metrics_ids = Arc::new( + buffer::MetricsIds::with_stride(64).inspect_err(at("metrics IDs"))?, + GFP_KERNEL, + )?; + let metrics = Arc::new( + KernelObject::new( + dev, + &render_global, + Placement::In(buffer::BlockGeometry::M5_COMPACT, mmu::UAT_PGSZ as u64), + metrics_ids.storage_bytes()?, + mmu::PROT_GPU_FW_SHARED_RW, + CpuMap::WriteCombined, + ) + .inspect_err(at("metrics"))?, + GFP_KERNEL, + )?; + let session_metrics = metrics.clone(); + let firmware = KBox::try_init( + kernel::try_init!(Firmware { + secondary: Some(secondary), + primary, + bridge: Bridge::unused(regs), + port: port::Port::new( + power::Demand::new(pdev.as_ref().as_raw())?, + shared.feed.clone(), + )?, + seen_submissions: 0, + rings: channel::Rings::new(shared.variant.graph)?, + recovery: recovery::State::new(), + effort: power::Effort::new(), + hold: power::Hold::new(), + governor: thermal::Governor::new(), + registration_power_release_pending: false, + variant: shared.variant, + queues <- runtime::Registry::with_waiters( + shared.clusters, + shared.deflake, + shared.descriptor_flags, + shared.variant, + Some(shared.queues.clone()), + Some(shared.renders.clone()), + ), + pool: object::Pool::new()?, + render_rings: kick::RenderRings::new()?, + render_ids: freelist::RenderIds::new()?, + init: initdata::InitData::new(), + _render_global: render_global, + _pm_metrics: session_metrics, + uat, + }), + GFP_KERNEL, + ) + .inspect_err(at("session"))?; + // Once a role runs, a stopped CPU is not proof that no engine still reads what it was + // given: a failed bootstrap keeps the session, nothing is freed or unmapped. A kept + // session's callbacks run this module's code, so the module is pinned first; once its + // removal has begun nothing is started and the session is freed unstarted. + retain_session().inspect_err(at("module pin"))?; + let mut retained = core::mem::ManuallyDrop::new(firmware); + let session = &mut **retained; + // M5 renders take 64-byte metrics records. + session.queues.metrics = metrics_ids; + session.primary.boot().inspect_err(at("primary boot"))?; + let secondary = session.secondary.as_mut().ok_or(EINVAL)?; + secondary.boot().inspect_err(at("secondary boot"))?; + // The firmware built its root and kernel-window tables while booting. + session.uat.invalidate_reserved_tables(); + let root_entries = session + .uat + .adopt_firmware_root_entries() + .inspect_err(at("root adoption"))?; + session + .init + .build_m5(dev, &session.uat, id, board) + .inspect_err(at("M5 graph"))?; + session + .init + .bind_m5_metrics(&session.uat, &metrics) + .inspect_err(at("metrics binding"))?; + // Publishers reach the mailbox, QoS table and recovery word through the port, as on + // G17P; M5's rings are the shared ones its InitData views name. + let mailbox: *const Mailbox = &*session.primary.rtkit; + session + .port + .attach( + session.init.port_views(mailbox)?, + session.bridge.regs.clone(), + shared.variant.registration, + !shared.variant.m5_commands, + ) + .inspect_err(at("port"))?; + session + .uat + .confirm_firmware_root_entries(root_entries) + .inspect_err(at("root confirmation"))?; + session.uat.mirror_secondary_root().inspect_err(at("secondary root"))?; + session + .bridge + .regs + .prepare_m5_firmware() + .inspect_err(at("M5 preparation"))?; + let ack = shared.variant.init_ack; + session + .primary + .init(session.init.root_va(Role::Primary)?, ack) + .inspect_err(at("primary InitData"))?; + dev_info!(dev.as_ref(), "M5 primary initdata acknowledged\n"); + let root_va = session.init.root_va(Role::Secondary)?; + let secondary = session.secondary.as_mut().ok_or(EINVAL)?; + secondary.init(root_va, ack).inspect_err(at("secondary InitData"))?; + dev_info!(dev.as_ref(), "M5 secondary initdata acknowledged\n"); + session.primary.notify(MSG_POWER_ON_DONE)?; + secondary.notify(MSG_POWER_ON_DONE)?; + Self::wait_for_opening(dev, &session.init, &session.primary, session.secondary.as_ref()) + .inspect_err(at("opening records"))?; + dev_info!(dev.as_ref(), "M5 firmware opening records retired\n"); + Ok(core::mem::ManuallyDrop::into_inner(retained)) + } + /// Starts both coprocessors and initializes their firmware. fn boot( pdev: &platform::Device>, @@ -1428,8 +1935,12 @@ impl Gpu { cfg: &'static Config, regs: regs::G17Resources, geometry: mmu::UatGeometry, + id: ®s::G17Id, shared: &Arc, ) -> Result> { + if cfg.profile.is_m5() { + return Self::boot_m5(pdev, dev, cfg, regs, geometry, id, shared); + } let uat = Arc::new( mmu::Uat::new( dev.into(), @@ -1448,8 +1959,8 @@ impl Gpu { let pm_metrics = KernelObject::new( dev, uat.kernel_lower_vm(), - Placement::At(cfg.pm_metrics.va), - cfg.pm_metrics.size, + Placement::At(cfg.pm_metrics()?.va), + cfg.pm_metrics()?.size, mmu::PROT_GPU_FW_SHARED_RW, CpuMap::WriteCombined, )?; @@ -1494,6 +2005,7 @@ impl Gpu { recovery: recovery::State::new(), effort: power::Effort::new(), hold: power::Hold::new(), + governor: thermal::Governor::new(), registration_power_release_pending: false, variant: shared.variant, queues <- runtime::Registry::with_waiters( @@ -1506,7 +2018,7 @@ impl Gpu { ), pool: object::Pool::new()?, render_rings: kick::RenderRings::new()?, - render_ids: Arc::new(freelist::RenderIds::new(), GFP_KERNEL)?, + render_ids: freelist::RenderIds::new()?, init: initdata::InitData::new(), _render_global: render_global, _pm_metrics: Arc::new(pm_metrics, GFP_KERNEL)?, @@ -1543,6 +2055,7 @@ impl Gpu { session.init.port_views(mailbox)?, session.bridge.regs.clone(), shared.variant.registration, + !shared.variant.m5_commands, )?; session.uat.mirror_secondary_root()?; if cfg.fabric_bridge { @@ -1653,6 +2166,11 @@ impl Gpu { self.shared.stop_queues(); self.shared.disable_timer(&self.shared.poll); self.shared.disable_timer(&self.shared.idle); + self.shared.disable_timer(&self.shared.governor); + if let Some(firmware) = (*self.shared.state.lock()).as_deref_mut() { + firmware.governor_restore(); + } + self.shared.actuators.lock().release(); self.shared.disable_work(&self.shared.grow); self.shared.disable_work(&self.shared.reclaim_work); self.shared.disable_work(&self.shared.work); @@ -1701,16 +2219,34 @@ impl gpu::Gpu for Gpu { u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_PER_COMMAND_INPUT_SYNCS) | u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_EXACT_PRIOR_BARRIERS) | u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_FRAGMENT_BARRIERS) - | u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_BOUNDED_AS_BUILD) | sync_output::FEATURE_CUMULATIVE_SIGNALS; let variant = self.shared.variant; + // The M5 command set has no AS-build flag: its validation refuses one. + if !variant.m5_commands { + params.features |= u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_BOUNDED_AS_BUILD); + } if variant.few_primitives { params.features |= u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_FEW_PRIMITIVES); } - if variant.ray_traversal { + if variant.render_ray_query { params.features |= u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_RENDER_RAY_QUERY); } + if variant.m5_commands { + let mut features = M5_FEATURES; + if !queue::render::m5_render_submission::depth_msaa_compression_enabled() { + features &= !M5_MSAA_COMPRESSION_FEATURES; + } + if crate::module_parameters::m5_mesh_storage_chunks.value() == 0 { + features &= !crate::file::DRM_ASAHI_FEATURE_TASK_MESH_STORAGE; + } + // Only where firmware waits need it (runtime::backend's + // firmware_sources_need_visible_completion), so userspace adds the flush only there. + if self.shared.m5_board.is_some_and(|board| board.completion_inputs()) { + features |= crate::file::DRM_ASAHI_FEATURE_COMPUTE_VISIBLE_COMPLETION; + } + params.features |= features; + } params.gpu_generation = self.cfg.gpu_generation as u32; params.gpu_variant = self.cfg.identity.gpu_variant as u32; params.gpu_revision = self.id.gpu_rev as u32; @@ -1752,7 +2288,7 @@ impl gpu::Gpu for Gpu { fn queue_limits(&self) -> Option { Some(uapi::drm_asahi_queue_limits { max_queues: kick::queue_limit(self.shared.variant.qid_floor), - max_in_flight_per_queue: hw::t8140::queues::RENDER_PAIRS_PER_OWNER as u32, + max_in_flight_per_queue: pairs::per_owner().into(), ..Default::default() }) } @@ -1769,9 +2305,23 @@ impl gpu::Gpu for Gpu { .uat .new_vm(self.ids.vm.next(), kernel_range)? .with_status()?; + if self.cfg.profile.is_m5() { + // The canonical arena the client's queues alias, the client-private one and the + // compact window, where the kernel places the parameter buffers' page lists, render + // targets, scratch and metrics records, stay reserved against userspace binds from + // creation, so no bind replaces a kernel alias the firmware reads. The client's own + // kernel range need not cover the compact window. + let ranges = firmware.uat.m5_client_reserved_ranges(); + let mut reserved = KVec::with_capacity(ranges.len() + 1, GFP_KERNEL)?; + for range in Iterator::chain(ranges.into_iter(), [buffer::BlockGeometry::M5_COMPACT]) { + reserved.push(range, GFP_KERNEL)?; + } + vm.install_driver_mappings(KVec::new(), reserved)?; + return Ok(vm); + } let mappings = firmware.init.user_aliases(&vm, &firmware._pm_metrics)?; let ranges = - hw::t8140::resources::client_reserved_ranges(self.cfg.pm_metrics.va).ok_or(EINVAL)?; + hw::t8140::resources::client_reserved_ranges(self.cfg.pm_metrics()?.va).ok_or(EINVAL)?; let mut reserved = KVec::with_capacity(ranges.len(), GFP_KERNEL)?; for range in ranges { reserved.push(range, GFP_KERNEL)?; @@ -1797,7 +2347,9 @@ impl gpu::Gpu for Gpu { self.geometry.user_usable_range(), self.shared.variant, )?; - let policy = scheduling_policy(priority)?; + let policy = if self.cfg.profile.is_m5() { + fw::m5::queue::scheduling_policy(priority)? + } else { scheduling_policy(priority)? }; let admission_started = now_ns(); let (backend, slot) = loop { let observed = self.shared.queues.observe(); @@ -1848,9 +2400,11 @@ impl gpu::Gpu for Gpu { let (mapping, window) = { let state = self.shared.state.lock(); let firmware = (*state).as_ref().ok_or(ENODEV)?; - // Firmware-only, like the other alias: only the firmware writes these stamps. - let windowed = bo - .map_range_into_range( + // The timestamp window is part of the older firmware graph, not M5's graph. + let windowed = if firmware.uat.is_m5() { + None + } else { + bo.map_range_into_range( firmware.uat.kernel_vm(), range.clone(), hw::t8140::dynamic::TIMESTAMP_WINDOW, @@ -1858,7 +2412,8 @@ impl gpu::Gpu for Gpu { mmu::PROT_FW_SHARED_RW, false, ) - .ok(); + .ok() + }; match windowed { Some(mapping) => (mapping, true), // A full window: this buffer's stamps are all host-written. @@ -1866,7 +2421,7 @@ impl gpu::Gpu for Gpu { bo.map_range_into_range( firmware.uat.kernel_vm(), range, - hw::t8140::dynamic::upper(firmware.uat.geometry().kernel_range()), + firmware.uat.kernel_dynamic_range(), mmu::UAT_PGSZ as u64, mmu::PROT_FW_SHARED_RW, false, diff --git a/drivers/gpu/drm/asahi/g17/object.rs b/drivers/gpu/drm/asahi/g17/object.rs index e60e4f376dd83..0b9854e84de2c 100644 --- a/drivers/gpu/drm/asahi/g17/object.rs +++ b/drivers/gpu/drm/asahi/g17/object.rs @@ -75,9 +75,13 @@ impl KernelObject { prot: mmu::Prot, cpu: CpuMap, ) -> Result { - let mut this = Self::backing(dev, size, cpu)?; + Self::backing(dev, size, cpu)?.placed(vm, placement, prot) + } + + /// Maps fresh backing into `vm`. + fn placed(mut self, vm: &mmu::Vm, placement: Placement, prot: mmu::Prot) -> Result { let mapping = match placement { - Placement::At(addr) => this.obj.map_at(vm, addr, prot, false)?, + Placement::At(addr) => self.obj.map_at(vm, addr, prot, false)?, Placement::In(range, align) => { let site = if range.start >= crate::hw::t8140::dynamic::LOWER.start && range.end <= crate::hw::t8140::dynamic::LOWER.end { @@ -85,21 +89,24 @@ impl KernelObject { } else { super::capacity::Site::UpperVa }; - this.obj.map_into_range(vm, range, align, prot, true).map_err(|error| { + self.obj.map_into_range(vm, range, align, prot, true).map_err(|error| { if error == ENOSPC { super::capacity::refused(site) } else { error } })? } }; - this.mapping = Some(mapping); - Ok(this) + self.mapping = Some(mapping); + Ok(self) } /// Allocates CPU-accessible backing whose GPU mappings will be installed later. pub(crate) fn backing(dev: &AsahiDrmDevice, size: usize, cpu: CpuMap) -> Result { - let mut obj = match cpu { + Self::with_object(match cpu { CpuMap::WriteBack => gem::new_kernel_object(dev, size)?, CpuMap::WriteCombined => gem::new_kernel_object_wc(dev, size)?, - }; + }, size) + } + + fn with_object(mut obj: gem::ObjectRef, size: usize) -> Result { let vmap = obj.vmap()?; let ptr = NonNull::new(vmap.addr.cast::()).ok_or(ENOMEM)?; // SAFETY: The mapping covers the whole object. @@ -229,6 +236,13 @@ impl KernelObject { self.mapping.take().ok_or(EINVAL) } + /// Publishes the page-table updates of the object's own mapping to every context naming its + /// VM's root (see [`mmu::KernelMapping::publish_to_contexts`]). + pub(crate) fn publish_to_contexts(&self) -> Result { + self.mapping.as_ref().ok_or(EINVAL)?.publish_to_contexts(); + Ok(()) + } + /// Returns the GPU address of the object. pub(crate) fn gpu_va(&self) -> u64 { self.mapping.as_ref().map_or(0, |mapping| mapping.iova()) @@ -338,7 +352,7 @@ impl Allocator<'_> { prot: mmu::Prot, cpu: CpuMap, ) -> Result { - let range = crate::hw::t8140::dynamic::upper(self.uat.geometry().kernel_range()); + let range = self.uat.kernel_dynamic_range(); KernelObject::new( self.dev, self.uat.kernel_vm(), @@ -361,13 +375,29 @@ impl Allocator<'_> { KernelObject::new( self.dev, self.uat.kernel_lower_vm(), - Placement::In(crate::hw::t8140::dynamic::LOWER, align), + Placement::In(self.uat.kernel_lower_dynamic_range(), align), size, prot, cpu, ) } + /// [`Self::lower`] write-combined backing that is not counted as kernel memory: the + /// caller charges it to a client ([`crate::memacct::Charge::user`]). + pub(crate) fn lower_user_charged( + &self, + size: usize, + align: u64, + prot: mmu::Prot, + ) -> Result { + KernelObject::with_object(gem::new_kernel_object_wc_user_charged(self.dev, size)?, size)? + .placed( + self.uat.kernel_lower_vm(), + Placement::In(self.uat.kernel_lower_dynamic_range(), align), + prot, + ) + } + /// Place fresh backing at one address in the canonical and both alias VMs. /// A client may retain mappings after their canonical reservation is freed. /// Retry only address collisions, keeping the same backing throughout. @@ -447,9 +477,11 @@ pub(crate) enum Family { WorkState = 1, QueueState = 2, QueueConfig = 3, + /// An M5 context's scheduler page, whose mapping the M5 firmware caches. + Scheduler = 4, } -const POOL_FAMILIES: usize = 4; +const POOL_FAMILIES: usize = 5; const POOL_SLOTS: usize = 64; const POOL_BYTES: usize = 128 * 1024 * 1024; diff --git a/drivers/gpu/drm/asahi/g17/pairs.rs b/drivers/gpu/drm/asahi/g17/pairs.rs new file mode 100644 index 0000000000000..c7515c1c76417 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/pairs.rs @@ -0,0 +1,47 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Render pairs one logical render queue may hold: two on the G17D (M5 Ultra), one elsewhere. +//! +//! A second pair lets one queue's depth run on two physical render rings. It comes only from +//! surplus: the QID budget ([`super::kick::render_reserve`]) keeps counting one pair per open +//! queue, admission claims a sibling only while a pair can still be built, and an unused +//! sibling is reassigned to an owner that needs a first pair. An owner's overlapping renders on +//! its two pairs are ordered in firmware ([`super::hazard::plan`]). +//! +//! A queue samples the count once, when it is created ([`per_owner`]): its scheduler credits, +//! render pool and pair limit all follow that one value. + +use core::sync::atomic::{AtomicU8, Ordering::Relaxed}; + +use crate::hw::t8140::queues::RENDER_PAIRS_MAX; + +/// G17D's count. +const G17D_PAIRS: u8 = RENDER_PAIRS_MAX as u8; + +static PAIRS: AtomicU8 = AtomicU8::new(1); + +/// The count for the probed board, set once at probe (one GPU per system). +pub(crate) fn select_board(board: &crate::g17::fw::m5::initdata::Board) { + PAIRS.store(if board.g17d_forms() { G17D_PAIRS } else { 1 }, Relaxed); +} + +/// Render pairs a logical render queue created now may hold. +pub(crate) fn per_owner() -> u8 { + PAIRS.load(Relaxed) +} + +/// Renders a busy pair of a two-pair owner takes before its next render goes to +/// an idle sibling: 1 keeps a queue at depth 1-2 on one pair. +const SIBLING_BACKLOG: u8 = 1; +static BACKLOG: AtomicU8 = AtomicU8::new(SIBLING_BACKLOG); + +/// A two-pair owner's next render stays on its least-loaded busy pair while +/// that pair holds at most this many renders. +pub(crate) fn sibling_backlog() -> u8 { + BACKLOG.load(Relaxed) +} + +#[cfg(CONFIG_KUNIT = "y")] +pub(crate) fn set_sibling_backlog(renders: u8) { + BACKLOG.store(renders, Relaxed); +} diff --git a/drivers/gpu/drm/asahi/g17/port.rs b/drivers/gpu/drm/asahi/g17/port.rs index 425eb6f517b71..be27fedaca625 100644 --- a/drivers/gpu/drm/asahi/g17/port.rs +++ b/drivers/gpu/drm/asahi/g17/port.rs @@ -188,13 +188,16 @@ impl Port { views: Views, regs: crate::regs::G17Resources, registration: super::fw::variant::QueueRegistration, + clamp_rx: bool, ) -> Result { // SAFETY: The session owns the mailbox and outlives every port user. let rx = unsafe { (*views.mailbox).rx_util_min_handle() }; + // Without `clamp_rx` (M5) no handle is attached and the receive thread keeps its + // default; rx_update() then returns at once. if self.views.populate(views) && self.regs.populate(regs) && self.registration.populate(registration) - && self.rx.populate(rx) + && (!clamp_rx || self.rx.populate(rx)) { Ok(()) } else { @@ -453,6 +456,16 @@ impl Port { self.qos.lock().cancel(&view, publication) } + /// Clears the QoS table rows of released QoS IDs (`QosIds::forget_released`); returns + /// those left because work still runs on them. + pub(super) fn qos_forget(&self, ids: u128) -> Result { + if ids == 0 { + return Ok(0); + } + let view = self.qos_view()?; + Ok(self.qos.lock().forget(&view, ids)) + } + pub(super) fn qos_complete(&self, owners: &[qos::Owner]) -> Result { let view = self.qos_view()?; self.qos.lock().complete(&view, owners)?; @@ -460,10 +473,10 @@ impl Port { Ok(()) } - /// Requests the receive thread's minimum utilization for the feed's current view of the GPU. - /// Called at every publication and completion: the feed's state lingers for a few - /// milliseconds, so after a pause the publication that ends it restores the default before - /// its completion arrives. + /// Requests the receive thread's minimum utilization for the feed's current view of the GPU, + /// where the port clamps it (not on M5, see attach()). Called at every publication and + /// completion: the feed's state lingers for a few milliseconds, so after a pause the + /// publication that ends it restores the default before its completion arrives. fn rx_update(&self) { let Some(rx) = self.rx.as_ref() else { return; diff --git a/drivers/gpu/drm/asahi/g17/power.rs b/drivers/gpu/drm/asahi/g17/power.rs index a2e293df91510..5c57f6a289440 100644 --- a/drivers/gpu/drm/asahi/g17/power.rs +++ b/drivers/gpu/drm/asahi/g17/power.rs @@ -40,6 +40,13 @@ pub(super) struct Effort { idle_since: Option, applied: u32, submissions: u64, + /// A host thread waits, or waited within [`IDLE_HOLD_NS`], for submitted work + /// ([`Self::note_waiters`]). + waited: bool, + /// The waits begun when the waiters were last noted. + seen_waits: u64, + /// When a waiter was last seen. + last_wait_ns: u64, } impl Effort { @@ -49,9 +56,36 @@ impl Effort { idle_since: None, applied: 0, submissions: 0, + waited: false, + seen_waits: 0, + last_wait_ns: 0, } } + pub(super) fn streaming(&self) -> bool { + self.streaming + } + + /// Whether a host thread waits for the work ([`Self::note_waiters`]). + pub(super) fn waited(&self) -> bool { + self.waited + } + + /// Notes the waiters on submitted work at `now_ns` ([`super::fence::waiters`]): a thread + /// waits now or began a wait since the last note. The note holds for [`IDLE_HOLD_NS`] after + /// the last waiter, across a submit-and-wait loop's turns between its waits. Returns whether + /// that changed. + pub(super) fn note_waiters(&mut self, now_ns: u64, (waiters, waits): (u32, u64)) -> bool { + if waiters > 0 || waits != self.seen_waits { + self.last_wait_ns = now_ns.max(1); + } + self.seen_waits = waits; + let waited = self.last_wait_ns != 0 && now_ns.saturating_sub(self.last_wait_ns) < IDLE_HOLD_NS; + let changed = waited != self.waited; + self.waited = waited; + changed + } + pub(super) fn submit(&mut self) { self.streaming = true; self.idle_since = None; @@ -84,6 +118,8 @@ impl Effort { let idle = now_ns.saturating_sub(since); if idle >= IDLE_HOLD_NS { self.streaming = false; + self.waited = false; + self.last_wait_ns = 0; self.idle_since = None; IdleAction::Clear } else { diff --git a/drivers/gpu/drm/asahi/g17/preparation.rs b/drivers/gpu/drm/asahi/g17/preparation.rs index c7d1761699a24..85994ab8e6c73 100644 --- a/drivers/gpu/drm/asahi/g17/preparation.rs +++ b/drivers/gpu/drm/asahi/g17/preparation.rs @@ -30,6 +30,10 @@ pub(super) struct Admission<'a> { check: &'a dyn Fn() -> Result, } +/// Bound for a compute publication's waits: a physical compute queue, or room on the shared +/// work ring, frees only as other owners' work retires or goes idle. +pub(super) const QUEUE_WAIT_NS: u64 = 30_000_000_000; + impl<'a> Admission<'a> { pub(super) fn new(deadline: u64, check: &'a dyn Fn() -> Result) -> Self { Self { deadline: Cell::new(deadline), started: super::now_ns(), check } diff --git a/drivers/gpu/drm/asahi/g17/qos.rs b/drivers/gpu/drm/asahi/g17/qos.rs index b36656b0174a7..a90fe51539cb1 100644 --- a/drivers/gpu/drm/asahi/g17/qos.rs +++ b/drivers/gpu/drm/asahi/g17/qos.rs @@ -237,6 +237,29 @@ impl Accounting { self.complete(view, &[ticket.owner]) } + /// Clears the scheduler and share rows of the QoS IDs in `ids`, which no context holds: a + /// released context's row otherwise keeps naming its scheduler page, which the pool may + /// already have given to another process's context. Rows of IDs with work in flight are + /// left alone and returned, for a later call once that work completed. + pub(crate) fn forget(&self, view: &View<'_>, ids: u128) -> u128 { + let mut left = 0; + for qos in 0..fw::QUEUES.min(128) { + if ids & (1u128 << qos) == 0 { + continue; + } + if self.in_flight.iter().zip(view.rows.iter()) + .any(|(&count, row)| count != 0 && usize::from(row[0].load(Ordering::Relaxed)) == qos) + { + left |= 1u128 << qos; + continue; + } + view.scheduler[qos].store(0, Ordering::Relaxed); + view.share[qos].store(0, Ordering::Relaxed); + } + fence(Ordering::SeqCst); + left + } + /// Preflights the entire set before clearing any final row. QID membership, /// scheduler mappings and monotonically increasing counts remain intact. pub(crate) fn complete(&mut self, view: &View<'_>, owners: &[Owner]) -> Result { diff --git a/drivers/gpu/drm/asahi/g17/queue/admission.rs b/drivers/gpu/drm/asahi/g17/queue/admission.rs index 9337ba9de281f..6fd8f13730c5c 100644 --- a/drivers/gpu/drm/asahi/g17/queue/admission.rs +++ b/drivers/gpu/drm/asahi/g17/queue/admission.rs @@ -11,7 +11,7 @@ use crate::g17::{ fw::kick::KickDependency, hazard::{self, Footprint, Plan}, - job::Packet, + job::{Order, Packet}, }; use crate::{hw::t8140::queues as cfg, mmu::MAX_EXECUTION_CONTEXTS}; use core::sync::atomic::{AtomicU32, Ordering}; @@ -107,12 +107,25 @@ impl State { fn exhausted(&self) -> bool { self.quarantined & Self::MASK == Self::MASK } + /// The owner's quarantined pairs, if any: `Some(true)` when every one of + /// them is spared (quarantined behind another VM's recovery). One + /// quarantined pair stops the owner on all of its pairs, so a healthy + /// sibling never bypasses it. + fn owner_quarantine(&self, owner: u64) -> Option { + let owned = (0..cfg::RENDER_SLOTS) + .filter(|slot| self.owners[*slot] == owner) + .fold(0u64, |mask, slot| mask | 1u64 << slot); + let quarantined = self.quarantined & owned; + (quarantined != 0).then_some(quarantined & !self.spared == 0) + } fn refusal(&self, owner: u64) -> Result { if self.exhausted() { return Err(EIO); } + if self.owner_quarantine(owner).is_some() { + return Err(EIO); + } match self.owners.iter().position(|id| *id == owner) { - Some(slot) if self.quarantined & (1u64 << slot) != 0 => Err(EIO), None if !self.owners[1..].contains(&0) && self.released & !self.active @@ -147,10 +160,28 @@ impl State { self.advance(slot); slot as u8 } - fn acquire(&mut self, owner: u64) -> Option { + fn acquire(&mut self, owner: u64, pairs: usize) -> Option { if owner == 0 { return None; } + // A shallow queue stays on its busy pair: a second pair pays off only + // once the first has a backlog (at depth 2 a switch on every render + // cost a 1080p fill about 45 us a submit on M5 Ultra). + if pairs > 1 { + let busy = (0..cfg::RENDER_SLOTS) + .filter(|slot| { + self.owners[*slot] == owner + && self.refs[*slot] != 0 + && self.active & (1u64 << slot) != 0 + && self.usable(owner, *slot) + }) + .min_by_key(|slot| (self.refs[*slot], u64::MAX - self.epochs[*slot])); + if let Some(slot) = busy.filter(|slot| self.refs[*slot] <= crate::g17::pairs::sibling_backlog()) { + self.refs[slot] += 1; + self.advance(slot); + return Some(slot as u8); + } + } // Prefer the owner's idle pair. A missing sibling can be claimed // only while this owner has no live kick, before constructing it. let idle = (0..cfg::RENDER_SLOTS) @@ -164,7 +195,7 @@ impl State { if !(0..cfg::RENDER_SLOTS) .any(|other| self.owners[other] == owner && self.refs[other] != 0) { - self.claim_sibling(owner); + self.claim_sibling(owner, pairs); } return Some(self.take_idle(slot, owner)); } @@ -203,11 +234,12 @@ impl State { }) .or_else(unowned)?; let first = self.take_idle(slot, owner); - self.claim_sibling(owner); + self.claim_sibling(owner, pairs); Some(first) } - fn claim_sibling(&mut self, owner: u64) { - if self.owners.iter().filter(|id| **id == owner).count() >= cfg::RENDER_PAIRS_PER_OWNER { + /// Claims another pair for an owner that may hold `pairs`. + fn claim_sibling(&mut self, owner: u64, pairs: usize) { + if self.owners.iter().filter(|id| **id == owner).count() >= pairs { return; } // A speculative sibling never takes a killed context's pair. @@ -238,6 +270,8 @@ impl State { /// The device lock separately proves the physical graph was never published. /// An open owner keeps another usable pair. Exclude every live borrower, /// including off-lock sibling construction; a closed owner needs no spare. + /// A spare goes only to an owner without a pair: a second pair comes only + /// from surplus, through claim_sibling. fn reassign_unused_spare( &mut self, slot: u8, @@ -249,8 +283,7 @@ impl State { || slot >= cfg::RENDER_SLOTS || owner.is_some_and(|owner| { owner == 0 - || self.owners.iter().filter(|value| **value == owner).count() - >= cfg::RENDER_PAIRS_PER_OWNER + || self.owners.contains(&owner) }) { return false; @@ -336,14 +369,11 @@ impl State { /// Whether `pick` would return a slot, without taking it. fn can_pick(&self, owner: u64, plan: Plan) -> bool { - let usable = |slot: usize| { - self.owners[slot] == owner - && (self.quarantined | self.kill_hold) & (1u64 << slot) == 0 - && self.refs[slot] < cfg::RENDER_DEPTH - }; + let usable = |slot: usize| self.usable(owner, slot); match plan { Plan::Wait => false, - Plan::Pair(slot) => usize::from(slot) < cfg::RENDER_SLOTS && usable(usize::from(slot)), + Plan::Pair(slot) => usable(usize::from(slot)), + Plan::Join(_) => (0..cfg::RENDER_SLOTS).any(usable), Plan::Any if owner == 0 => false, Plan::Any if self.owners.contains(&owner) => (0..cfg::RENDER_SLOTS).any(usable), Plan::Any => { @@ -356,26 +386,36 @@ impl State { } } - fn pick(&mut self, owner: u64, plan: Plan) -> Option { + fn pick(&mut self, owner: u64, pairs: usize, plan: Plan) -> Option { match plan { - Plan::Any => self.acquire(owner), + Plan::Any => self.acquire(owner, pairs), Plan::Wait => None, - Plan::Pair(slot) => { - let index = usize::from(slot); - if self.owners.get(index) != Some(&owner) - || (self.quarantined | self.kill_hold) & (1u64 << slot) != 0 - || self.refs[index] >= cfg::RENDER_DEPTH - { - return None; - } - if self.refs[index] == 0 { - Some(self.take_idle(index, owner)) - } else { - self.refs[index] += 1; - self.advance(index); - Some(slot) - } - } + Plan::Pair(slot) => self.join(owner, usize::from(slot)), + // The preferred pair, else the owner's least-loaded other pair: + // the publication barrier orders the overlap from there. + Plan::Join(slot) => self.join(owner, usize::from(slot)).or_else(|| { + let other = (0..cfg::RENDER_SLOTS) + .filter(|other| *other != usize::from(slot) && self.usable(owner, *other)) + .min_by_key(|other| (self.refs[*other], u64::MAX - self.epochs[*other]))?; + self.join(owner, other) + }), + } + } + fn usable(&self, owner: u64, slot: usize) -> bool { + self.owners.get(slot) == Some(&owner) + && (self.quarantined | self.kill_hold) & (1u64 << slot) == 0 + && self.refs[slot] < cfg::RENDER_DEPTH + } + fn join(&mut self, owner: u64, index: usize) -> Option { + if !self.usable(owner, index) { + return None; + } + if self.refs[index] == 0 { + Some(self.take_idle(index, owner)) + } else { + self.refs[index] += 1; + self.advance(index); + Some(index as u8) } } @@ -501,17 +541,14 @@ impl Pool { } /// The owner's pair is quarantined behind another VM's recovery (spared) and drains. pub(crate) fn spared_quarantined(&self, owner: u64) -> bool { - let state = self.state.lock(); - state - .owners - .iter() - .position(|id| *id == owner) - .is_some_and(|slot| state.quarantined & state.spared & (1u64 << slot) != 0) + self.state.lock().owner_quarantine(owner) == Some(true) } + /// For an owner that may hold `pairs` render pairs. pub(crate) fn acquire_render( self: &Arc, owner: u64, + pairs: usize, request: Option>, deadline_ns: u64, check: impl Fn() -> Result, @@ -532,22 +569,12 @@ impl Pool { if state.exhausted() { return Err(EIO); } - // The first retained pair's quarantine is not bypassed through its + // A retained pair's quarantine is not bypassed through its // healthy sibling. Reclamation wakes this bounded admission wait. - if state - .owners - .iter() - .position(|id| *id == owner) - .is_some_and(|slot| state.quarantined & (1u64 << slot) != 0) - { + if let Some(spared) = state.owner_quarantine(owner) { // Behind an innocent pair's quarantine (another VM wedged the GPU) wait for the // pair to drain until the caller's deadline, which the caller extends while it // drains (RECOVERY_WAIT_MS); a guilty pair's publication fails after a second. - let spared = state - .owners - .iter() - .position(|id| *id == owner) - .is_some_and(|slot| state.spared & (1u64 << slot) != 0); if now >= quarantine_deadline && !spared { return Err(EIO); } @@ -570,7 +597,7 @@ impl Pool { } else { state.plan(owner, request, ticket) }; - if let Some(slot) = state.pick(owner, plan) { + if let Some(slot) = state.pick(owner, pairs, plan) { state.forget_waiter(ticket.take()); let record = request.and_then(|request| state.record(owner, Some(slot), 0, request)); @@ -613,6 +640,7 @@ impl Pool { pub(crate) fn try_acquire_render( self: &Arc, owner: u64, + pairs: usize, request: Option>, ) -> Result> { if owner == 0 { @@ -623,7 +651,7 @@ impl Pool { return Ok(None); } let plan = state.plan(owner, request, None); - let Some(slot) = state.pick(owner, plan) else { + let Some(slot) = state.pick(owner, pairs, plan) else { return Ok(None); }; let record = request.and_then(|request| state.record(owner, Some(slot), 0, request)); @@ -640,12 +668,7 @@ impl Pool { if owner == 0 || state.refusal(owner).is_err() { return false; } - if state - .owners - .iter() - .position(|id| *id == owner) - .is_some_and(|slot| state.quarantined & (1u64 << slot) != 0) - { + if state.owner_quarantine(owner).is_some() { return false; } let plan = state.plan(owner, request, None); @@ -735,7 +758,7 @@ mod accounting_tests { fn every_logical_owner_fits_without_speculative_spares() { let mut state = State::new(); for owner in 1..=u64::from(MAX_EXECUTION_CONTEXTS) { - let slot = state.acquire(owner).unwrap(); + let slot = state.acquire(owner, 1).unwrap(); assert_eq!(state.owners.iter().filter(|id| **id == owner).count(), 1); state.release(slot); } @@ -746,7 +769,7 @@ mod accounting_tests { #[test] fn close_does_not_reuse_an_active_lease() { let mut state = State::new(); - let slot = state.acquire(1).unwrap(); + let slot = state.acquire(1, 1).unwrap(); state.released |= 1u64 << slot; assert!(!state.reassign_unused_spare(slot, Some(1), None)); state.release(slot); @@ -759,7 +782,7 @@ mod accounting_tests { #[test] fn warming_quarantine_and_exit_hold_exclude_reassignment() { let mut state = State::new(); - let primary = state.acquire(1).unwrap(); + let primary = state.acquire(1, 1).unwrap(); // A retained legacy sibling still obeys all quarantine/kill guards. state.owners[2] = 1; let spare = state.borrow_warm_sibling(1, primary).unwrap(); @@ -805,6 +828,19 @@ impl QueueSlot { pub(crate) fn pool(&self) -> &Arc { &self.pool } + /// Gives back a slot no queue was published in, for a creator that sampled the waiters' + /// domain as `observed` before claiming it. Returns the sample that creator waits on next + /// ([`Availability::notify_own`]): its own return still wakes the other waiters, but not + /// itself. + /// + /// [`Availability::notify_own`]: crate::g17::availability::Availability::notify_own + pub(crate) fn cancel(self, observed: u64) -> u64 { + let this = core::mem::ManuallyDrop::new(self); + // SAFETY: `this` is never used or dropped again, so the pool reference moves out once. + let pool = unsafe { core::ptr::read(&this.pool) }; + pool.queues.fetch_sub(1, Ordering::AcqRel); + pool.waiters.as_ref().map_or(observed, |waiters| waiters.notify_own(observed)) + } } impl Drop for QueueSlot { fn drop(&mut self) { @@ -879,11 +915,20 @@ impl Pool { /// Under the device mutex, which orders every publication: the newest /// published fragment kick of each other pair whose live same-VM render /// overlaps `footprint`. Quarantined pairs are skipped as in admission. + /// + /// That includes the owner's own other pair, for its renders that `prefix` + /// (the render's queue prefix) does not already order: an owner's renders + /// on its two pairs run on different queues (see [`hazard::plan`]). An + /// owner publishes in submission order (the backlog keeps inline + /// publication behind queued commands, and the ordered run_job worker + /// publishes one command at a time), so its earlier renders already carry + /// their fragment kick here. pub(crate) fn publication_barriers( &self, owner: u64, slot: u8, footprint: &Footprint, + prefix: Option, barriers: &mut [KickDependency; hazard::PUBLICATION_BARRIERS], ) -> Result { let state = self.state.lock(); @@ -894,7 +939,7 @@ impl Pool { continue; }; if other == slot - || entry.owner == owner + || (entry.owner == owner && Order::contains(prefix, entry.sequence)) || state.quarantined & (1u64 << other) != 0 || !footprint.same_vm(&entry.footprint) || !footprint.overlaps(&entry.footprint) @@ -1011,6 +1056,35 @@ impl Pool { } } +#[kunit_tests(asahi_g17_queue_slot_cancel)] +mod queue_slot_cancel_tests { + use super::*; + use crate::g17::availability::Availability; + + #[test] + fn a_creator_does_not_wake_itself_by_giving_back_its_slot() -> Result { + let waiters = Availability::new()?; + let pool = Pool::with_waiters(MAX_EXECUTION_CONTEXTS, Some(waiters.clone()), None)?; + let observed = waiters.observe(); + let slot = pool.reserve_queue()?; + let next = slot.cancel(observed); + assert_eq!(pool.queues(), 0); + // The return advanced the domain, so other waiters wake, but the creator's next + // sample is current: its wait registers instead of returning at once. + assert_eq!(next, waiters.observe()); + assert_ne!(next, observed); + assert!(waiters.wait(next)?.is_some()); + // A return by anyone else after the sample still ends the creator's wait. + let observed = waiters.observe(); + let slot = pool.reserve_queue()?; + drop(pool.reserve_queue()?); + assert_eq!(slot.cancel(observed), observed); + assert!(waiters.wait(observed)?.is_none()); + waiters.notify(); + Ok(()) + } +} + #[kunit_tests(asahi_g17_fault_admission)] mod fault_capacity_tests { use super::*; @@ -1019,7 +1093,7 @@ mod fault_capacity_tests { fn a_guilty_failure_makes_a_spared_quarantine_guilty() { let pool = Pool::new(MAX_EXECUTION_CONTEXTS).unwrap(); let queue = pool.reserve_queue().unwrap(); - let lease = pool.acquire_render(1, None, u64::MAX, || Ok(())).unwrap(); + let lease = pool.acquire_render(1, 1, None, u64::MAX, || Ok(())).unwrap(); let bit = 1u64 << lease.slot(); assert!(!pool.spared_quarantined(1)); lease.quarantine(true); @@ -1045,7 +1119,7 @@ mod fault_capacity_tests { let pool = Pool::new(MAX_EXECUTION_CONTEXTS).unwrap(); for owner in 1..=512 { let queue = pool.reserve_queue().unwrap(); - let lease = pool.acquire_render(owner, None, u64::MAX, || Ok(())).unwrap(); + let lease = pool.acquire_render(owner, 1, None, u64::MAX, || Ok(())).unwrap(); lease.quarantine(false); pool.release_owner(owner); let bit = 1u64 << lease.slot(); @@ -1066,7 +1140,7 @@ mod fault_capacity_tests { #[test] fn published_exit_kill_holds_survive_close_and_drain() { let pool = Pool::new(MAX_EXECUTION_CONTEXTS).unwrap(); - let lease = pool.acquire_render(1, None, u64::MAX, || Ok(())).unwrap(); + let lease = pool.acquire_render(1, 1, None, u64::MAX, || Ok(())).unwrap(); let bit = 1u64 << lease.slot(); pool.release_owner(1); drop(lease); @@ -1081,7 +1155,7 @@ mod fault_capacity_tests { #[test] fn consumed_scheduler_release_ends_the_kill_hold() { let pool = Pool::new(MAX_EXECUTION_CONTEXTS).unwrap(); - let lease = pool.acquire_render(1, None, u64::MAX, || Ok(())).unwrap(); + let lease = pool.acquire_render(1, 1, None, u64::MAX, || Ok(())).unwrap(); let slot = lease.slot(); let bit = 1u64 << slot; pool.release_owner(1); @@ -1095,7 +1169,7 @@ mod fault_capacity_tests { // The killed context's pair is never reassigned as a spare, and a new owner takes // a new graph before it. assert!(!pool.reassign_unused_spare(slot, Some(1), Some(2))); - let next = pool.acquire_render(2, None, u64::MAX, || Ok(())).unwrap(); + let next = pool.acquire_render(2, 1, None, u64::MAX, || Ok(())).unwrap(); assert_ne!(next.slot(), slot); drop(next); pool.release_retained_owners_after_processor_stop(); @@ -1116,9 +1190,9 @@ mod fault_capacity_tests { // One pair released without a kill. state.killed &= !(1u64 << 7); } - let first = pool.acquire_render(1, None, u64::MAX, || Ok(())).unwrap(); + let first = pool.acquire_render(1, 1, None, u64::MAX, || Ok(())).unwrap(); assert_eq!(first.slot(), 7); - let second = pool.acquire_render(2, None, u64::MAX, || Ok(())).unwrap(); + let second = pool.acquire_render(2, 1, None, u64::MAX, || Ok(())).unwrap(); assert_eq!(usize::from(second.slot()), cfg::RENDER_SLOTS - 1); assert_eq!(pool.state.lock().killed & (1u64 << second.slot()), 0); } @@ -1136,11 +1210,11 @@ mod fault_capacity_tests { } } // With room in the queue table a new owner builds a pair in an unowned slot. - let built = pool.acquire_render(1, None, u64::MAX, || Ok(())).unwrap(); + let built = pool.acquire_render(1, 1, None, u64::MAX, || Ok(())).unwrap(); assert!(usize::from(built.slot()) >= 8); // Without room it takes the killed pair released longest ago. pool.note_build_room(false); - let reused = pool.acquire_render(2, None, u64::MAX, || Ok(())).unwrap(); + let reused = pool.acquire_render(2, 1, None, u64::MAX, || Ok(())).unwrap(); assert_eq!(reused.slot(), 7); assert_eq!(pool.state.lock().killed & (1u64 << 7), 0); // It claims no sibling: neither another killed pair nor an unowned slot. @@ -1165,12 +1239,164 @@ mod shared_depth_tests { #[test] fn a_full_pair_waits_for_retirement_instead_of_allocating_a_spare() { let mut state = State::new(); - let first = state.acquire(1).unwrap(); - for _ in 1..cfg::RENDER_DEPTH { assert_eq!(state.acquire(1), Some(first)); } - assert_eq!(state.acquire(1), None); + let first = state.acquire(1, 1).unwrap(); + for _ in 1..cfg::RENDER_DEPTH { assert_eq!(state.acquire(1, 1), Some(first)); } + assert_eq!(state.acquire(1, 1), None); assert_eq!(state.owners.iter().filter(|id| **id == 1).count(), 1); state.release(first); - assert_eq!(state.acquire(1), Some(first)); + assert_eq!(state.acquire(1, 1), Some(first)); + } +} + +#[kunit_tests(asahi_g17_owner_pairs)] +mod owner_pair_tests { + use super::*; + use crate::g17::fw::kick::KickTimestamp; + + fn kick(qid: u8, timestamp: u64) -> KickDependency { + KickDependency::new(qid, KickTimestamp::new(timestamp).unwrap()).unwrap() + } + + #[test] + fn a_full_preferred_pair_moves_to_the_owners_other_pair() { + let mut state = State::new(); + state.owners[1] = 1; + state.owners[2] = 1; + for _ in 0..cfg::RENDER_DEPTH { + assert_eq!(state.pick(1, 1, Plan::Pair(1)), Some(1)); + } + assert!(!state.can_pick(1, Plan::Pair(1))); + assert_eq!(state.pick(1, 1, Plan::Pair(1)), None); + assert!(state.can_pick(1, Plan::Join(1))); + assert_eq!(state.pick(1, 1, Plan::Join(1)), Some(2)); + // With one pair a preference is the pair. + state.owners[2] = 0; + assert!(!state.can_pick(1, Plan::Join(1))); + assert_eq!(state.pick(1, 1, Plan::Join(1)), None); + } + + #[test] + fn any_quarantined_pair_stops_its_owner() { + let mut state = State::new(); + state.owners[1] = 1; + state.owners[2] = 1; + assert!(state.owner_quarantine(1).is_none()); + state.quarantined |= 1u64 << 2; + assert_eq!(state.owner_quarantine(1), Some(false)); + assert!(state.refusal(1).is_err()); + state.spared |= 1u64 << 2; + assert_eq!(state.owner_quarantine(1), Some(true)); + state.quarantined |= 1u64 << 1; + assert_eq!(state.owner_quarantine(1), Some(false)); + } + + #[test] + fn an_uncovered_own_render_on_the_other_pair_is_a_barrier() { + let pool = Pool::new(MAX_EXECUTION_CONTEXTS).unwrap(); + let mut footprint = Footprint::new(1, false); + footprint.push_vm(0, 0x1000); + let footprint = Arc::new(footprint, GFP_KERNEL).unwrap(); + { + let mut state = pool.state.lock(); + for (slot, sequence, timestamp) in [(1u8, 4u64, 7u64), (1, 5, 9), (2, 6, 3)] { + state + .footprints + .push( + InFlight { + owner: 1, + sequence, + slot: Some(slot), + ticket: 0, + footprint: footprint.clone(), + fragment: Some(kick(10 + slot, timestamp)), + }, + GFP_KERNEL, + ) + .unwrap(); + } + } + let mut barriers = [KickDependency::ZERO; hazard::PUBLICATION_BARRIERS]; + // On pair 2: pair 1's newest overlapping fragment kick, never its own. + assert_eq!(pool.publication_barriers(1, 2, &footprint, None, &mut barriers).unwrap(), 1); + assert!(barriers[0] == kick(11, 9)); + assert_eq!(pool.publication_barriers(1, 2, &footprint, Some(4), &mut barriers).unwrap(), 1); + assert!(barriers[0] == kick(11, 9)); + // The queue prefix already orders them. + assert_eq!(pool.publication_barriers(1, 2, &footprint, Some(5), &mut barriers).unwrap(), 0); + // Another owner's render is a barrier whatever the prefix. + assert_eq!(pool.publication_barriers(3, 2, &footprint, Some(9), &mut barriers).unwrap(), 1); + assert_eq!(pool.publication_barriers(3, 3, &footprint, Some(9), &mut barriers).unwrap(), 2); + } +} + +#[kunit_tests(asahi_g17_two_pairs)] +mod two_pair_tests { + use super::*; + + #[test] + fn a_two_pair_owner_spreads_its_depth_over_a_surplus_sibling() { + crate::g17::pairs::set_sibling_backlog(0); + let mut state = State::new(); + let first = state.acquire(1, 2).unwrap(); + // The sibling is claimed with the first pair, from surplus. + let sibling = (0..cfg::RENDER_SLOTS) + .find(|slot| *slot != usize::from(first) && state.owners[*slot] == 1) + .unwrap(); + // With the first pair busy, the next render takes the idle sibling, then + // the least-loaded of the two. + assert_eq!(usize::from(state.acquire(1, 2).unwrap()), sibling); + let third = state.acquire(1, 2).unwrap(); + assert!(third == first || usize::from(third) == sibling); + for _ in 3..2 * usize::from(cfg::RENDER_DEPTH) { + assert!(state.acquire(1, 2).is_some()); + } + assert_eq!(state.refs[usize::from(first)], cfg::RENDER_DEPTH); + assert_eq!(state.refs[sibling], cfg::RENDER_DEPTH); + assert_eq!(state.acquire(1, 2), None); + assert_eq!(state.owners.iter().filter(|id| **id == 1).count(), 2); + crate::g17::pairs::set_sibling_backlog(1); + } + + #[test] + fn a_shallow_two_pair_queue_stays_on_its_busy_pair() { + crate::g17::pairs::set_sibling_backlog(1); + let mut state = State::new(); + let first = state.acquire(1, 2).unwrap(); + // At depth 2 the second render stacks on the busy pair. + assert_eq!(state.acquire(1, 2), Some(first)); + // With a backlog there, the third takes the idle sibling. + let third = state.acquire(1, 2).unwrap(); + assert_ne!(third, first); + assert_eq!(state.owners[usize::from(third)], 1); + // Retired back to one render, the pair keeps the next one. + state.release(first); + state.release(third); + assert_eq!(state.acquire(1, 2), Some(first)); + } + + #[test] + fn a_sibling_needs_room_to_build_and_never_takes_a_spare() { + let mut state = State::new(); + state.build_room = false; + // A released pair of a closed owner is reused for the first pair; no + // other free slot is claimed without room to build. + state.owners[5] = 9; + state.released |= 1u64 << 5; + assert_eq!(state.acquire(1, 2), Some(5)); + assert_eq!(state.owners.iter().filter(|id| **id == 1).count(), 1); + // An unused spare goes only to an owner without a pair. + state.owners[6] = 3; + state.released |= 1u64 << 6; + assert!(!state.reassign_unused_spare(6, Some(3), Some(1))); + assert!(state.reassign_unused_spare(6, Some(3), Some(2))); + } + + #[test] + fn a_one_pair_owner_claims_no_sibling() { + let mut state = State::new(); + let first = state.acquire(1, 1).unwrap(); + assert_eq!(state.owners.iter().filter(|id| **id == 1).count(), 1); + assert_eq!(state.acquire(1, 1), Some(first)); } } @@ -1191,7 +1417,7 @@ mod deadline_tests { state.active = State::MASK; } let start = crate::g17::now_ns(); - let result = pool.acquire_render(1, None, start + 20_000_000, || Ok(())); + let result = pool.acquire_render(1, 1, None, start + 20_000_000, || Ok(())); assert!(matches!(result, Err(error) if error == EBUSY)); assert!(crate::g17::now_ns() - start < 500_000_000); assert!(pool.state.lock().footprints.is_empty()); @@ -1209,7 +1435,7 @@ mod deadline_tests { state.active = State::MASK; } let start = crate::g17::now_ns(); - let result = pool.acquire_render(1, None, start + 2_000_000_000, || { + let result = pool.acquire_render(1, 1, None, start + 2_000_000_000, || { if crate::g17::now_ns() - start >= 10_000_000 { Err(ECANCELED) } else { Ok(()) } }); assert!(matches!(result, Err(error) if error == ECANCELED)); @@ -1220,7 +1446,7 @@ mod deadline_tests { #[test] fn cancellation_precedes_available_capacity() { let pool = Pool::new(MAX_EXECUTION_CONTEXTS).unwrap(); - assert!(matches!(pool.acquire_render(1, None, u64::MAX, || Err(ECANCELED)), + assert!(matches!(pool.acquire_render(1, 1, None, u64::MAX, || Err(ECANCELED)), Err(error) if error == ECANCELED)); assert_eq!(pool.state.lock().active, 0); } diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 9f847eb383931..3e9d2002dafc3 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -4,6 +4,8 @@ //! installed graph; the device retains it until both firmware processors have stopped. pub(crate) mod exit; +mod activation; +mod configuration; pub(crate) mod retirement; use crate::g17::{ @@ -12,12 +14,13 @@ use crate::g17::{ fw::{ channels::ControlRecord, compute::{ComputeProgram, OperandState, COMPUTE_DESCRIPTOR_SIZE}, + m5::initdata::Board, queue::*, variant::PointerLayout, }, object::{Allocator, CpuMap, Family, KernelObject, Pool, PooledObject}, }; -use crate::{hw::t8140::dynamic, mmu}; +use crate::mmu; use core::sync::atomic::{fence, Ordering}; use kernel::{prelude::*, sync::Arc}; use retirement::{Cursors, RECORD_SLOTS}; @@ -90,7 +93,7 @@ pub(crate) struct Binding { preempt: mmu::KernelMapping, usage: mmu::KernelMapping, operand: mmu::KernelMapping, - _queue_context: mmu::KernelMapping, + _queue_context: Option, context: Arc, } @@ -106,6 +109,8 @@ impl DescriptorRing { /// Firmware reads every descriptor through a cached kernel mapping, as it /// does for render. The pool keeps that mapping until processor stop; the /// hardware reads the same pages through an uncached canonical low alias. + /// On M5 the kernel mapping, in the upper half every execution context + /// includes, is firmware-only. pub(crate) fn new(alloc: &Allocator<'_>, objects: &Arc) -> Result { let size = DESCRIPTORS * COMPUTE_DESCRIPTOR_SIZE; let object = objects.allocate( @@ -113,12 +118,12 @@ impl DescriptorRing { Family::Descriptor, size, size, - mmu::PROT_GPU_FW_SHARED_RW, + if alloc.uat.is_m5() { mmu::PROT_FW_SHARED_RW } else { mmu::PROT_GPU_FW_SHARED_RW }, CpuMap::WriteCombined, )?; let low = object.alias_in( alloc.uat.kernel_lower_vm(), - dynamic::LOWER, + alloc.uat.kernel_lower_dynamic_range(), mmu::UAT_PGSZ as u64, mmu::PROT_GPU_SHARED_RW, )?; @@ -129,14 +134,16 @@ impl DescriptorRing { /// Firmware-shared storage of one compute QID, separate from its common kick ring. /// `ring` is `None` while the queue is vacant (idle, no owner, pool given back). struct Graph { + m5: bool, ring: Option, preempt: KernelObject, usage: KernelObject, operand: KernelObject, support: KernelObject, + m5_timestamps: Option>, graph: PooledObject, - context_low: KernelObject, - context_high: KernelObject, + context_low: Option, + context_high: Option, /// Announcements, which stay uncached for the firmware. records: KernelObject, /// Configuration, predecessor and completion-wait records at the same @@ -148,6 +155,8 @@ struct Graph { pointers: PointerLayout, /// Layout of the operand-state buffer for the GPU's cores. operand_state: &'static OperandState, + /// The M5 board, whose part selects the M5 descriptor words. + board: Option<&'static Board>, program: ComputeProgram, } @@ -158,7 +167,9 @@ impl Graph { context: &Context, pointers: PointerLayout, operand_state: &'static OperandState, + board: Option<&'static Board>, program: ComputeProgram, + m5: bool, ) -> Result { let page = mmu::UAT_PGSZ as u64; let prot = mmu::PROT_GPU_FW_SHARED_RW; @@ -167,12 +178,16 @@ impl Graph { let preempt = alloc.kernel(0x4000, page, prot, cpu)?; let usage = alloc.kernel(mmu::UAT_PGSZ, page, prot, cpu)?; let operand = alloc.lower(operand_state.size, page, prot, cpu)?; - let mut support = alloc.kernel(0xc000, page, prot, cpu)?; - let support_va = support.gpu_va(); - support.write(0, support_va + page)?; - support.write(0x100, support_va + page + 4)?; - support.write(0x110, 0x50u32)?; - support.write(mmu::UAT_PGSZ + 4, 1u32)?; + // The M5 support page is firmware-only; the GPU has no access to it. + let support_prot = if m5 { mmu::PROT_FW_SHARED_RW } else { prot }; + let mut support = alloc.kernel(if m5 { mmu::UAT_PGSZ } else { 0xc000 }, page, support_prot, cpu)?; + if !m5 { + let support_va = support.gpu_va(); + support.write(0, support_va + page)?; + support.write(0x100, support_va + page + 4)?; + support.write(0x110, 0x50u32)?; + support.write(mmu::UAT_PGSZ + 4, 1u32)?; + } let mut graph = objects.allocate( alloc, Family::QueueState, @@ -183,14 +198,6 @@ impl Graph { )?; let graph_va = graph.gpu_va(); graph.initialize::(0, |view| { - view.queue = QueueRecord::new(&QueueRecordArgs { - pointers_va: graph_va + CURSORS as u64, - items_va: graph_va + ITEMS as u64, - job_list_va: context.work_head_va(), - scheduler_va: context.scheduler_va(), - owner_pid: context.owner_pid(), - policy: context.policy(), - }); view.pointers = PointerBlock::new(&pointers)?; view.jobs = JobListHead::new(graph_va + 0x200); Ok(()) @@ -199,9 +206,29 @@ impl Graph { *view = PointerBlock::new(&pointers)?; Ok(()) })?; - let context_low = alloc.lower(CONTEXT_SIZE, page, prot, cpu)?; - let context_high = alloc.kernel(CONTEXT_SIZE, page, prot, cpu)?; + let record = QueueRecordArgs { + pointers_va: graph_va + CURSORS as u64, + items_va: graph_va + ITEMS as u64, + job_list_va: context.work_head_va(), + scheduler_va: context.scheduler_va(), + owner_pid: context.owner_pid(), + policy: context.policy(), + }; + // One queue record, the firmware's own: the M5 record is shorter than the G17P one the + // layout's slot holds, and the zeroed layout leaves the rest of the slot zero. + if m5 { + graph.write(0, crate::g17::fw::m5::queue::QueueRecord::new(&record)?)?; + } else { + graph.write(0, QueueRecord::new(&record))?; + } + let (context_low, context_high) = if m5 { (None, None) } else { + (Some(alloc.lower(CONTEXT_SIZE, page, prot, cpu)?), + Some(alloc.kernel(CONTEXT_SIZE, page, prot, cpu)?)) + }; let records = alloc.kernel(RECORD_SLOTS * RECORD_STRIDE, page, prot, cpu)?; + let m5_timestamps = if m5 { + Some(Arc::new(alloc.kernel(RECORD_SLOTS * 16, page, prot, cpu)?, GFP_KERNEL)?) + } else { None }; let cached_records = objects.allocate( alloc, Family::QueueConfig, @@ -211,11 +238,13 @@ impl Graph { cpu, )?; Ok(Self { + m5, ring: Some(ring), preempt, usage, operand, support, + m5_timestamps, graph, context_low, context_high, @@ -225,6 +254,7 @@ impl Graph { reclaimed: 0, pointers, operand_state, + board, program, }) } @@ -240,20 +270,20 @@ impl Graph { return Err(EFAULT); } let vm = context.vm(); - vm.install_job_context_aliases()?; + if !self.m5 { vm.install_job_context_aliases()?; } let descriptors = self.descriptor_alias(vm)?; - let prot = mmu::PROT_GPU_FW_SHARED_RW; + let prot = if self.m5 { mmu::PROT_GPU_SHARED_RW } else { mmu::PROT_GPU_FW_SHARED_RW }; let page = mmu::UAT_PGSZ as u64; + let private = mmu::client_private_range(self.m5); let preempt = self .preempt - .alias_in(vm, dynamic::CLIENT_LOWER, page, prot)?; - let usage = self.usage.alias_in(vm, dynamic::CLIENT_LOWER, page, prot)?; + .alias_in(vm, private.clone(), page, prot)?; + let usage = self.usage.alias_in(vm, private.clone(), page, prot)?; let operand = self .operand - .alias_in(vm, dynamic::CLIENT_LOWER, page, prot)?; - let queue_context = self - .context_low - .alias_in(vm, dynamic::CLIENT_LOWER, page, prot)?; + .alias_in(vm, private.clone(), page, prot)?; + let queue_context = self.context_low.as_ref() + .map(|context| context.alias_in(vm, private.clone(), page, prot)).transpose()?; Ok(Binding { _kick_alias: kick_alias, pool_alias: Some(pool_alias), @@ -268,6 +298,15 @@ impl Graph { /// Caller has excluded publication and proved the retained graph idle. fn set_owner(&mut self, context: &Context) -> Result { + if self.m5 { + let pointer = self.graph.pointer(0, + core::mem::size_of::())? + .cast::(); + // SAFETY: The checked packed record stays mapped. The device mutex + // and idle witness exclude access by another owner during this update. + return unsafe { (*pointer).set_compute_owner(context.work_head_va(), + context.scheduler_va(), context.owner_pid(), context.policy()) }; + } let pointer = self .graph .pointer(0, core::mem::size_of::())? @@ -313,6 +352,15 @@ impl Graph { /// Detaches the descriptor ring of an idle vacant queue; the caller frees it /// off-lock. The job list and the context item only name consumed /// descriptors at this point. + /// Clears the client-visible state the last owner's work left (preemption + /// buffer, usage page, operand state) before another VM is given views of + /// it. The caller proved the queue idle. + fn clear_client_state(&mut self) { + self.preempt.bytes_mut().fill(0); + self.usage.bytes_mut().fill(0); + self.operand.bytes_mut().fill(0); + fence(Ordering::SeqCst); + } fn vacate_ring(&mut self) -> Option { self.ring.take() } @@ -323,9 +371,13 @@ impl Graph { self.ring = Some(ring); Ok(()) } - fn ensure_record_slot(&self, slot: usize) -> Result { + /// `cursors`: this publication's snapshot (M5), else a fresh read. + fn ensure_record_slot(&self, slot: usize, cursors: Option) -> Result { if let Some(end) = *self.record_ends.get(slot).ok_or(EINVAL)? { - let cursors = self.cursors()?; + let cursors = match cursors { + Some(cursors) => cursors, + None => self.cursors()?, + }; if !cursors.consumed(end, cursors.count)? { return Err(EAGAIN); } @@ -376,7 +428,8 @@ impl Graph { } use crate::g17::{ - completion::Deferred, context::WorkStateLease, freelist::ComputePool, job::Packet, kick, qos, + completion::Deferred, context::WorkStateLease, + freelist::{ComputeCapacity, ComputePool, PoolDescriptor}, job::Packet, kick, qos, recovery, status::VmStatus, }; use retirement::{Batch, Observation, Recordless, Ticket, IN_FLIGHT}; @@ -476,6 +529,9 @@ struct Previous> { binding: Option, context: C, publication: Option<(u64, u64)>, + /// The queue moved to another VM. The previous owner's views were unmapped at the move, + /// so a rollback leaves the queue detached instead of restoring them. + moved: bool, } impl Previous { /// The exact drain witness covers the replacement publication even when @@ -509,14 +565,15 @@ pub(crate) struct PoolBacking { } impl PoolBacking { - pub(crate) fn new(alloc: &Allocator<'_>, id: kick::Id, blocks: usize) -> Result { - let pool = ComputePool::new(alloc, id.qid(), blocks)?; + pub(crate) fn new(alloc: &Allocator<'_>, id: kick::Id, required: ComputeCapacity, + client: Option<&Arc>) -> Result { + let pool = ComputePool::new(alloc, id.qid(), required, client)?; Ok(Self { pool, id }) } /// Fresh page-pool row data, installed once while the QID is still unpublished. - pub(crate) fn pool_descriptor(&self) -> Result<(u16, u64)> { - Ok((self.pool.id(), self.pool.page_list_va()?)) + pub(crate) fn pool_descriptor(&self) -> Result { + self.pool.descriptor() } } @@ -562,12 +619,15 @@ pub(crate) struct Queue { pool_phase: PoolPhase, release_sent_ns: Option, growing: bool, - growth_blocks: usize, + /// What the growth in progress must provide. + growth: Option, /// A killed queue was recycled: its pool/ring must be vacated and re-armed /// fresh before any owner binds it (set by `recycle_killed`, cleared by `rearm`). needs_fresh_backing: bool, /// Monotonic time the last owner released this queue (vacate order). released_at: u64, + /// Monotonic time the watchdog quarantined this queue, zero otherwise. + timed_out_at: u64, /// Free-list generation of the last published configuration. free_list_generation: u64, /// `None` after an exited owner was detached; a new owner binds afresh. @@ -583,11 +643,18 @@ pub(crate) struct Queue { replay_retirement: bool, replay_error: Option, replay_prepared: bool, + /// Since when the replay front waits for its render consumers (see compute_replay). + consumer_wait_since: Option, exit: Option, /// Since when the kill of this failed queue's context waits for other contexts' compute, /// and whether that wait was reported. kill_wait: Option<(u64, bool)>, installed: bool, + /// The owner re-arming this vacant queue: only it builds the fresh backing, so concurrent + /// waiters do not each build a pool for the same queue. + rearm_for: Option, + configuration: Option, + m5_install: Option, /// Bit `p` set once a work slot of priority class `p` named this queue. outer_started: u8, published_config: Option, @@ -630,11 +697,13 @@ impl Queue { context: Arc, pointers: PointerLayout, operand_state: &'static OperandState, + board: Option<&'static Board>, program: ComputeProgram, + m5: bool, ) -> Result { let Backing { mut pool, mut kick } = backing; let (active, replays) = (Fifo::new()?, Fifo::new()?); - let mut graph = Graph::new(alloc, objects, &context, pointers, operand_state, program)?; + let mut graph = Graph::new(alloc, objects, &context, pointers, operand_state, board, program, m5)?; let kick_alias = kick.map_client(context.vm())?; let pool_alias = pool.map_client(context.vm())?; let binding = graph.bind(context.clone(), kick_alias, pool_alias)?; @@ -647,9 +716,10 @@ impl Queue { pool_phase: PoolPhase::Populated, release_sent_ns: None, growing: false, - growth_blocks: 0, + growth: None, needs_fresh_backing: false, released_at: 0, + timed_out_at: 0, free_list_generation: 0, binding: Some(binding), previous: None, @@ -662,9 +732,13 @@ impl Queue { replay_retirement: false, replay_error: None, replay_prepared: false, + consumer_wait_since: None, exit: None, kill_wait: None, installed: false, + rearm_for: None, + configuration: None, + m5_install: None, outer_started: 0, published_config: None, config_cache: None, @@ -772,10 +846,22 @@ impl Queue { !self .context() .is_some_and(|bound| Arc::ptr_eq(bound, context)) - || (self.active.len == 0 && self.replays.len == 0) + || (self.active.len == 0 && self.replays.len == 0 && self.configuration.is_none()) } pub(crate) fn in_flight(&self) -> bool { - self.active.len != 0 && !self.quarantined + (self.active.len != 0 || self.configuration.is_some()) && !self.quarantined + } + /// Per-queue consumer wait (see compute_replay). + pub(crate) fn consumer_wait(&mut self, now: u64) -> u64 { + now.saturating_sub(*self.consumer_wait_since.get_or_insert(now)) + } + pub(crate) fn clear_consumer_wait(&mut self) { + self.consumer_wait_since = None; + } + /// Cold M5 installation publishes one configuration before the first kick. + /// A prepared claim prevents reassignment while these credits are owned. + pub(crate) fn outer_slots(&self) -> u32 { + if self.graph.m5 && self.m5_install.is_none() { 2 } else { 1 } } /// The oldest command is a published long job (ray traversal or AS build) whose /// completion record the driver has not seen yet. @@ -799,7 +885,14 @@ impl Queue { self.active.front().map(|active| crate::g17::timeout::Head { vm: active.packet.context.vm().id(), published: active.published, - finished: active.observed.is_some(), + // An M5 completion record can precede the command's writes: only the stamp + // proves it finished. + finished: if self.graph.m5 { + active.ticket.as_ref().is_some_and(|ticket| self.graph.stamp() + .is_ok_and(|stamp| retirement::stamp_covers(stamp, ticket.kick))) + } else { + active.observed.is_some() + }, failed: self.quarantined, }) } @@ -862,7 +955,7 @@ impl Queue { /// A completion hint can require a second visibility/recordless pass even /// when firmware has no more events to send. Do not poll idle or failed queues. pub(crate) fn retirement_polling(&self) -> bool { - !self.quarantined && self.retire_pending + self.configuration.is_some() || (!self.quarantined && self.retire_pending) } /// A release has no completion event: keep observing the control consumer @@ -1030,6 +1123,7 @@ impl Queue { mut complete_qos: impl FnMut(qos::Owner) -> Result, defer: &mut impl FnMut(Deferred) -> Result, ) -> Result { + if recovery_state == 0 { self.retire_configuration(epoch)?; } if self.quarantined || !self.selected(masks) { return Ok(()); } @@ -1079,7 +1173,7 @@ impl Queue { recordless, self.last_end, next_end, - record_aged, + record_aged && !self.graph.m5, )?; if timestamps.is_some() && !retirement::stamp_covers(stamp, ticket.kick) { retirement::note_stampless(self.qid(), ticket.kick); @@ -1113,10 +1207,17 @@ impl Queue { ); } self.retire_pending = true; - defer(Deferred::Retired( - front.packet.completion.clone(), - Ok([timestamps[0], timestamps[1], 0, 0]), - ))?; + let completion = front.packet.completion.clone(); + let producers = front.packet.check_firmware_producers(); + defer(match producers { + Ok(()) if self.graph.m5 => Deferred::RetiredCompute( + completion, + [timestamps[0], timestamps[1], 0, 0], + front.observed.map(|record| record.timestamps), + ), + Ok(()) => Deferred::Retired(completion, Ok([timestamps[0], timestamps[1], 0, 0])), + Err(error) => Deferred::Retired(completion, Err(error)), + })?; self.retire_pending = false; self.submitted = submitted; let active = self.active.pop().ok_or(EIO)?; @@ -1217,21 +1318,43 @@ impl Queue { )? { return Ok(()); } + let unexecuted = if self.graph.m5 { + // M5 command words stay zero even for executed commands. The + // front needs its exact zero-duration ring record; a later + // command, chained to its predecessor by the kick parent, must + // not show a record with a real duration. A render consumer of + // this work (firmware dependency) is checked by the replay pass: + // it republishes only while every render of the owner is parked + // or idle and none executed since recovery. + self.active.iter().enumerate().all(|(index, active)| { + !active.packet.completion.signalled() + && match active.observed { + // Only reached for spared work quarantined after a + // recovery: every retained command crossed it. + Some(record) => { + record.timestamps[0] != 0 && retirement::unexecuted(record, true) + } + None => index != 0, + } + }) + } else { + !self + .active + .iter() + .any(|active| active.packet.completion.has_firmware_consumer()) + && self.active.iter().all(|active| { + active + .packet + .completion + .replay_witness_words() + .is_some_and(|words| words[0] == words[1]) + }) + }; let candidate = self.spared_deferred && self.spared_quarantine && !self.released && self.owner == Some(newest.owner) - && !self - .active - .iter() - .any(|active| active.packet.completion.has_firmware_consumer()) - && self.active.iter().all(|active| { - active - .packet - .completion - .replay_witness_words() - .is_some_and(|words| words[0] == words[1]) - }); + && unexecuted; if candidate && !replay { return Ok(()); } @@ -1266,6 +1389,7 @@ impl Queue { if self.spared_quarantine { self.spared_quarantine = false; self.quarantine_error = None; + self.timed_out_at = 0; self.submitted = 0; self.kick.clear_parent_after_recovery(); self.quarantined = false; @@ -1340,6 +1464,7 @@ impl Queue { /// Caller already stopped both processors. No firmware owner can access retained mappings. pub(crate) fn stopped(&mut self, defer: &mut impl FnMut(Deferred) -> Result) -> Result { + if let Some(configuration) = self.configuration.take() { configuration.release(); } while let Some(packet) = self.replays.front() { defer(Deferred::Retired(packet.completion.clone(), Err(EIO)))?; drop(self.replays.pop()); @@ -1617,6 +1742,7 @@ impl Queue { return false; } self.installed + && self.configuration.is_none() // A pool release in flight must be witnessed before any successor. && !matches!(self.pool_phase, PoolPhase::ReleaseSent(_)) // A recycled killed queue is offered only once its backing is vacant (re-arm path). @@ -1662,19 +1788,49 @@ impl Queue { binding: None, context: old.ok_or(EIO)?, publication: None, + moved: false, }); } else { - let kick_alias = self.kick.map_client(context.vm())?; - let pool_alias = self.pool.map_client(context.vm())?; - let binding = self.graph.bind(context.clone(), kick_alias, pool_alias)?; + // Another VM: the pool must be fresh backing (EPERM otherwise). The queue's state + // pages belong to one VM at a time. The previous owner's views (kick ring, + // preemption, usage and operand state, descriptors, queue context) are unmapped + // first, so its GPU work can no longer read or write what the new owner's work + // leaves there; then the pages are cleared and mapped into the new VM. The queue + // is idle (`reusable`) and the previous owner's pool release was consumed (fresh + // backing), so the firmware no longer uses those views. The previous context, + // whose scheduler state the queue may still run until the new owner's first + // command, stays in `Previous`. + if !self.pool.serves(context.vm()) { + return Err(EPERM); + } + drop(self.binding.take()); + let moved = (|| -> Result { + self.graph.clear_client_state(); + self.kick.clear_ring()?; + let kick_alias = self.kick.map_client(context.vm())?; + let pool_alias = self.pool.map_client(context.vm())?; + self.graph.bind(context.clone(), kick_alias, pool_alias) + })(); + let binding = match moved { + Ok(binding) => binding, + Err(error) => { + // Left detached: the record still names the previous owner's pages. + if let Some(old) = old.as_ref() { + self.retained_owner = Some(old.retain_firmware_pages()); + } + self.needs_fresh_backing = true; + return Err(error); + } + }; self.graph.set_owner(&context)?; - let old_binding = self.binding.replace(binding); + self.binding = Some(binding); // A detached queue (exited owner already dropped) has nothing to supersede; // the new owner's record fields replace the retained pages' references. self.previous = old.map(|old| Previous { - binding: old_binding, + binding: None, context: old, publication: None, + moved: true, }); self.retained_owner = None; } @@ -1687,6 +1843,7 @@ impl Queue { if same_vm { 2 } else { 1 }, ); self.owner = Some(owner); + self.rearm_for = None; self.unpublished_binding = true; self.released = false; self.retirement_ready = false; @@ -1703,7 +1860,7 @@ impl Queue { return Ok(true); } self.released = true; - if self.active.len == 0 && !self.quarantined { + if self.active.len == 0 && !self.quarantined && self.configuration.is_none() { if let Some(previous) = self.previous.as_ref().filter(|p| p.publication.is_none()) { if let Err(error) = self.graph.set_owner(&previous.context) { self.quarantined = true; @@ -1711,7 +1868,13 @@ impl Queue { return Err(error); } let mut previous = self.previous.take().ok_or(EIO)?; - if let Some(binding) = previous.binding.take() { + if previous.moved { + // The record names the previous owner again, whose views went at the + // move: the queue is left detached, as for an exited owner. + self.binding = None; + self.retained_owner = Some(previous.context.retain_firmware_pages()); + self.needs_fresh_backing = true; + } else if let Some(binding) = previous.binding.take() { self.binding = Some(binding); } else { self.binding.as_mut().ok_or(EIO)?.context = previous.context; @@ -1749,6 +1912,8 @@ impl Queue { } let binding = self.binding.take()?; self.retained_owner = Some(binding.context.retain_firmware_pages()); + // The backing served the exited VM only: give it back for fresh backing. + self.needs_fresh_backing = true; Some(binding) } /// An idle retained queue whose USC backing may be given back. @@ -1773,12 +1938,33 @@ impl Queue { pub(crate) fn needs_fresh_backing(&self) -> bool { self.needs_fresh_backing } + /// Whether this queue's pool backing may be bound for `vm`. + pub(crate) fn pool_serves(&self, vm: &mmu::Vm) -> bool { + self.pool.serves(vm) + } + /// Gives a released queue's backing back for fresh backing before reuse. + pub(crate) fn require_fresh_backing(&mut self) { + self.needs_fresh_backing = true; + } + /// A pool release is in flight, or a queue waits for one, that ends with a + /// vacant queue ready for re-arming. + pub(crate) fn vacancy_pending(&self) -> bool { + !self.growing && (self.pool_release_pending() + || (self.recycled_pending() && self.pool_phase == PoolPhase::Populated)) + } /// The firmware acknowledged the kill of this queue's closed owner (`ContextKilled`) - /// and every killed command was settled: return the QID to service. The item - /// ring is reset to empty at the firmware's consumer cursor; the USC pool and - /// descriptor ring are vacated and re-armed fresh before the next owner binds. + /// and every killed command was settled. The QID stays killed in the firmware + /// until a queue configuration with its QoS update set binds it to a scheduler + /// page other than the one the firmware has for it. A new owner's first + /// publication carries such a configuration, but its command is retired + /// unexecuted on the still-killed queue, with a zero-duration record. + /// + /// On M5 the queue is installed again instead: the next owner's first + /// publication sends the configuration alone and waits until the firmware + /// consumed it (`install_m5`), so the kill is cleared before any command. The + /// USC pool and descriptor ring are vacated and re-armed fresh as before. pub(crate) fn recycle_killed(&mut self) -> Result { - if !self.installed + if self.configuration.is_some() || !self.installed || !self.exit.as_ref().is_some_and(|exit| exit.released()) || self.active.len != 0 || self.replays.len != 0 @@ -1787,6 +1973,15 @@ impl Queue { return Ok(false); } self.recycle_settled()?; + if self.graph.m5 { + // Start over as a fresh install: no published configuration and kick + // count zero, so the next publication installs the queue first. Kick + // timestamps keep counting, so its stamps stay ordered. + self.m5_install = None; + self.published_config = None; + self.outer_started = 0; + self.ordinal = 0; + } Ok(true) } /// A quarantined queue whose owner closed and whose every retained command was @@ -1818,6 +2013,7 @@ impl Queue { self.exit = None; self.quarantined = false; self.quarantine_error = None; + self.timed_out_at = 0; self.failure_status_pending = false; self.retired_by_teardown = false; self.spared_quarantine = false; @@ -1844,6 +2040,18 @@ impl Queue { pub(crate) fn pool_vacant(&self) -> bool { self.pool_phase == PoolPhase::Vacant } + /// No other owner is re-arming this queue. + pub(crate) fn rearm_open(&self, owner: u64) -> bool { + self.rearm_for.is_none_or(|reserved| reserved == owner) + } + pub(crate) fn reserve_rearm(&mut self, owner: u64) { + self.rearm_for = Some(owner); + } + pub(crate) fn release_rearm(&mut self, owner: u64) { + if self.rearm_for == Some(owner) { + self.rearm_for = None; + } + } /// Publishes the release of this idle queue's free list. The caller holds the /// device mutex and rings the control doorbell afterwards. pub(crate) fn vacate(&mut self, init: &crate::g17::initdata::InitData) -> Result { @@ -1902,7 +2110,7 @@ impl Queue { &mut self, pages: crate::g17::freelist::Pages, ring: DescriptorRing, - ) -> Result<(u16, u64)> { + ) -> Result { if !self.can_rearm() { return Err(EAGAIN); } @@ -1911,10 +2119,10 @@ impl Queue { self.pool_phase = PoolPhase::Populated; self.release_sent_ns = None; self.needs_fresh_backing = false; - Ok((self.pool.id(), self.pool.page_list_va()?)) + self.pool.descriptor() } - pub(crate) fn pool_fits(&self, required: usize) -> bool { - !self.growing && self.pool_phase == PoolPhase::Populated && self.pool.fits(required) + pub(crate) fn pool_fits(&self, required: ComputeCapacity) -> bool { + !self.growing && self.pool_phase == PoolPhase::Populated && self.pool.fits_capacity(required) } /// Only a capacity increase waits for this queue's previously published work. @@ -1922,7 +2130,7 @@ impl Queue { pub(crate) fn begin_growth( &mut self, init: &crate::g17::initdata::InitData, - blocks: usize, + required: ComputeCapacity, ) -> Result { if self.growing { return Err(EAGAIN); @@ -1949,7 +2157,7 @@ impl Queue { self.release_sent_ns = None; } self.growing = true; - self.growth_blocks = blocks; + self.growth = Some(required); Ok(self.installed) } @@ -1958,9 +2166,10 @@ impl Queue { init: &crate::g17::initdata::InitData, pages: &mut Option, alias: &mut Option, - blocks: usize, ) -> Result { - if !self.growing || blocks < self.growth_blocks { + // The fresh backing must provide what the growth was begun for. + let target = self.growth.filter(|_| self.growing).ok_or(EAGAIN)?; + if !pages.as_ref().ok_or(EINVAL)?.fits_capacity(target) { return Err(EAGAIN); } if let PoolPhase::ReleaseSent(cursor) = self.pool_phase { @@ -1977,13 +2186,13 @@ impl Queue { let binding = self.binding.as_mut().ok_or(EFAULT)?; // No ownership moves before every fallible descriptor/header operation. // The old release is consumed; no queue can use this row until commit. - init.initialize_compute_pool_blocks(self.pool.id(), fresh.page_list_va(), blocks)?; + init.install_compute_pool(fresh.descriptor(self.pool.id())?)?; self.pool.replace_pages(pages)?; let old_alias = core::mem::replace(&mut binding.pool_alias, alias.take()); self.pool_phase = PoolPhase::Populated; self.release_sent_ns = None; self.growing = false; - self.growth_blocks = 0; + self.growth = None; Ok(Vacated { _alias: old_alias, _pages: pages.take(), @@ -2007,7 +2216,7 @@ impl Queue { return Ok(None); } self.growing = false; - self.growth_blocks = 0; + self.growth = None; self.release_owner().map(Some) } @@ -2050,6 +2259,7 @@ impl Queue { let result = self.quarantine(ETIMEDOUT, defer); if fresh { self.spared_quarantine = return_spared; + self.timed_out_at = now_ns().max(1); } result } @@ -2240,6 +2450,7 @@ pub(crate) trait Host { fn publish_outer(&mut self, priority: u8, slot: &crate::g17::fw::channels::WorkSlot) -> Result; fn notify(&mut self, message: u64) -> Result; fn note_submission(&mut self) -> Result; + fn submit_m5(&mut self, submission: &crate::g17::fw::m5::queue::HardwareSubmission) -> Result; } impl Queue { @@ -2274,6 +2485,11 @@ impl Queue { }; let (epoch, addresses) = self.publication_preflight(&packet, host)?; let crate::g17::command::Validated::Compute { + matrix_ops: requested_matrix_ops, + cdm_auxiliary_allocation, + plan, + m5_intersection, + m5_scratch, intersection, scratch, usc_base, @@ -2286,20 +2502,28 @@ impl Queue { else { return Err(EINVAL); }; + let m5_scratch = *m5_scratch; + if !self.graph.m5 && (*requested_matrix_ops || *cdm_auxiliary_allocation != 0) { return Err(EINVAL); } + if self.graph.m5 { + // Validation refuses a sampler heap on M5, and M5 completion waits ride the + // kick's dependencies (publish_compute_inputs). + if *sampler_count != 0 || !native.is_empty() { return Err(EINVAL); } + if plan.is_some_and(|plan| !self.pool.fits(Some(plan.geometry))) { return Err(EINVAL); } + self.install_m5(&packet, host)?; + } let work = packet.completion.work_state()?; let work_word = work.word(); if work_word == u32::MAX { return Err(EINVAL); } let work_node = work.node_va()?; - let timestamp_va = packet.completion.compute_timestamp_va()?; let kick_timestamp = self.kick.timestamp(); let kick_parent = self.kick.parent(); let qid = self.qid(); let ordinal = self.ordinal.checked_add(1).ok_or(EOVERFLOW)?; let (descriptor_high, descriptor_low) = self.graph.descriptor_vas(kick_timestamp.slot() as u8)?; - let first = !self.installed && self.ordinal == 0; + let first = self.ordinal == 0 && (self.graph.m5 || !self.installed); // The first prototype only waits on an installed consumer channel. // A retry without a published source falls back to completion on CPU. if first && !native.is_empty() { @@ -2312,7 +2536,27 @@ impl Queue { || dependencies .iter() .any(|dependency| dependency.qid() != qid); - item_capacity(self.graph.cursors()?, entry_signal)?; + // On M5 one read of the item-ring cursors serves the whole publication: the producer is + // the host's and only this publication moves it, and the consumer only advances, so the + // snapshot is conservative for the room, record-slot and reclaim checks. + let mut snapshot = None; + if self.graph.m5 { + // The configuration and its notification are the only items an M5 kick appends. + let cursors = self.graph.cursors()?; + if cursors.count != ITEM_RING_ENTRIES { + return Err(EIO); + } + if cursors.producer >= cursors.count || cursors.consumer >= cursors.count { + return Err(ERANGE); + } + let free = (cursors.consumer + cursors.count - cursors.producer - 1) % cursors.count; + if free < 2 { + return Err(ENOSPC); + } + snapshot = Some(cursors); + } else { + item_capacity(self.graph.cursors()?, entry_signal)?; + } if !native.is_empty() { let cursors = self.graph.cursors()?; let free = (cursors.consumer + cursors.count - cursors.producer - 1) % cursors.count; @@ -2320,10 +2564,15 @@ impl Queue { return Err(EAGAIN); } } - let record_slot = (!first).then_some(self.ordinal as usize % RECORD_SLOTS); + let record_slot = (self.graph.m5 || !first).then_some(self.ordinal as usize % RECORD_SLOTS); if let Some(slot) = record_slot { - self.graph.ensure_record_slot(slot)?; + self.graph.ensure_record_slot(slot, snapshot)?; } + let timestamp_va = if self.graph.m5 { + packet.completion.bind_m5_compute_timestamps( + self.graph.m5_timestamps.as_ref().ok_or(EINVAL)?, + record_slot.ok_or(EINVAL)? * 16)? + } else { packet.completion.compute_timestamp_va()? }; let args = ComputeArgs { program: self.graph.program, intersection: *intersection, @@ -2363,20 +2612,75 @@ impl Queue { }; self.free_list_generation = u64::from(packet.context.scheduler_generation()); let policy = packet.context.policy(); - let ring = 1u8 << policy.priority(); - let config = self.queue_config(&identity, first, args.context_id, policy, &packet)?; + let activation = activation::Publication::new( + self.outer_started, policy.priority(), self.graph.m5); + let config = self.queue_config(&identity, first, activation.refresh, + args.context_id, policy, &packet)?; let cached = self .config_cache .and_then(|(config, at)| (at == epoch).then_some(config)); - let send_config = first || super::render::publication::required(cached, config); + let send_config = self.graph.m5 || first || super::render::publication::required(cached, config); let announce = KickAnnounce::new(qid, DataMaster::Compute, kick_timestamp, policy.priority())?; let predecessor = KickPredecessor::new(qid, kick_parent)?; + let matrix_ops = self.graph.m5 && *requested_matrix_ops; + let m5_board = if self.graph.m5 { Some(self.graph.board.ok_or(EINVAL)?) } else { None }; + // Every earlier kick of this queue has retired, so the parent has completed. + let parent_retired = self.active.len == 0 + && self.replays.len == 0 + && !self.quarantined + && !self.awaiting_witness; + let parent_class = m5_board + .map(|board| crate::g17::fw::m5::compute::parent_dependency(board, parent_retired)); + let m5_args = m5_board.zip(parent_class).map(|(board, parent_class)| { + let tuple = crate::g17::fw::m5::compute::kick_tuple(board, parent_class); + crate::g17::fw::m5::compute::DescriptorArgs { + registers: crate::g17::fw::m5::compute::RegisterArgs { + matrix_ops, + scratch: m5_scratch, + intersection: *m5_intersection, + auxiliary_allocation: *cdm_auxiliary_allocation, + preempt_va: addresses.preempt, cdm_va: *cdm_va, operand_va: addresses.operand, + usage_va: addresses.usage, free_list_slot: u32::from(self.pool.id()), + context_id: args.context_id, work_state: work_word, + object_ids: 0x0000_0001_0000_0001, + resource_policy: plan.map_or(0xf5_0040_0020, |p| u64::from(p.pressure) << 32 | 0x0040_0020), + preempt_state_offset: tuple.preempt_state_offset, + // The board's core layout sizes the operand state, as on G17P. + operand_region_bytes: self.graph.operand_state.bank_bytes(), + operand_aux_offset: self.graph.operand_state.tail_offset(), + operand_bytes: self.graph.operand_state.size, resource_control: plan.map_or(0x54b, |p| p.resource_flags), + dispatch_control: crate::g17::fw::m5::compute::dispatch_control(board, tuple.dispatch_base), + auxiliary_control: tuple.auxiliary_control, + }, + descriptor_va: descriptor_low, + // M5 Pro's descriptors carry the ordinal as the submission sequence too, as macOS's do. + submit_sequence: if board.completion_inputs() { u64::from(self.ordinal) } else { 0 }, + work_state_va: work_node, + cdm_end_va: *cdm_end_va, stamp_va: [args.stamp_va, args.aux_stamp_va], + stamp_value: crate::g17::fw::stamp_value(kick_timestamp.get() as u32), qid: u32::from(qid), + kick_timestamp: kick_timestamp.get(), + completion_ordinal: crate::g17::fw::m5::compute::completion_ordinal(board, self.ordinal), + status_va: args.status_va, timestamp_va: [timestamp_va, timestamp_va + 8], + shared_control_va: self.pool.control_va(), event_metadata: [0;16], + usage_fw_va: self.graph.usage.gpu_va(), completion_identity: 1, + }}); + // Explicit dependencies take the board's explicit class; the implicit parent, last, + // takes its own ([`crate::g17::fw::m5::compute::parent_dependency`]). + let mut m5_dependency_types = [m5_board.map_or( + crate::g17::fw::m5::compute::COMPUTE_PIPELINED_DEPENDENCY, + crate::g17::fw::m5::compute::explicit_dependency); + crate::g17::fw::kick::QID_MAX as usize + 1]; + if let (Some(class), Some(parent)) = (parent_class, m5_dependency_types.get_mut(dependencies.len())) { + *parent = class; + } let kick_args = KickArgs { qid, timestamp: kick_timestamp, parent: kick_parent, + ancestor: None, barriers: dependencies, + dependency_types: self.graph.m5.then_some(&m5_dependency_types[..dependencies.len() + 1]), descriptor_va: descriptor_high, queue_va: self.graph.graph.gpu_va(), qos: KickQos { @@ -2384,9 +2688,17 @@ impl Queue { class: COMPUTE_QOS_CLASS, }, mcache: None, - event_mask: COMPUTE_KICK_EVENT_MASK, - register_arrays: ComputeDescriptor::register_bindings(descriptor_low, *scratch, self.graph.program)?, - scs_enabled: scratch.enabled(), + event_mask: match m5_board { + Some(board) => crate::g17::fw::m5::compute::kick_event_mask(board, identity.context)?, + None => COMPUTE_KICK_EVENT_MASK, + }, + register_arrays: if self.graph.m5 { + crate::g17::fw::m5::compute::bindings(descriptor_low, m5_scratch, matrix_ops)? + } else { + ComputeDescriptor::register_bindings(descriptor_low, *scratch, self.graph.program)? + }, + scs_enabled: if self.graph.m5 { m5_scratch.enabled() } else { scratch.enabled() }, + compute_matrix: matrix_ops, priority: policy.priority(), }; let published = host.next_compute_publication()?; @@ -2411,17 +2723,16 @@ impl Queue { let result = (|| -> Result { work.add_submitted_kicks(1)?; accounted = true; - self.graph - .ring - .as_mut() - .ok_or(EAGAIN)? - .object - .initialize::( - kick_timestamp.slot() % DESCRIPTORS * COMPUTE_DESCRIPTOR_SIZE, - |descriptor| descriptor.write_zeroed(&args), - )?; - if first { - self.graph.context_high.write( + let object = &mut self.graph.ring.as_mut().ok_or(EAGAIN)?.object; + let offset = kick_timestamp.slot() % DESCRIPTORS * COMPUTE_DESCRIPTOR_SIZE; + if let Some(m5_args) = &m5_args { + object.initialize::(offset, + |descriptor| descriptor.write_zeroed(m5_args))?; + } else { + object.initialize::(offset, |descriptor| descriptor.write_zeroed(&args))?; + } + if !self.graph.m5 && first { + self.graph.context_high.as_mut().ok_or(EINVAL)?.write( 0x200, crate::g17::fw::compute::QueueContextItem::new( descriptor_high, @@ -2429,18 +2740,19 @@ impl Queue { qid, ), )?; - } else { + } else if !self.graph.m5 { // The retained context item remains at index one; only its descriptor changes. - self.graph.context_high.write(0x210, descriptor_high)?; + self.graph.context_high.as_mut().ok_or(EINVAL)?.write(0x210, descriptor_high)?; let cursors = self.graph.cursors()?; let _ = self.graph.recycle(cursors); } + if let Some(cursors) = snapshot { self.graph.recycle(cursors)?; } let config_va = if send_config { Some(self.graph.record(record_slot, CONFIG, 0xc0, config)?) } else { None }; - let cursors = self.graph.cursors()?; + let cursors = match snapshot { Some(cursors) => cursors, None => self.graph.cursors()? }; let mut wait_vas = KVec::with_capacity(native.len(), GFP_KERNEL)?; for (index, wait) in native.iter().enumerate() { let record = wait.bytes( @@ -2458,7 +2770,26 @@ impl Queue { let kick_ring_va = self.kick.low_va(); let prepared_kick = self.kick.prepare_entry(&kick_args)?; let mut producer; - if first { + if self.graph.m5 { + let target = self.pool.reserve_submission()?; + let qos = host.qos_publish(qos::Owner { qid, qos: identity.qos, data_master: 2 }, + identity.scheduler, policy).or_else(|error| { + self.pool.cancel_unpublished(target)?; Err(error) + })?; + self.active.back_mut().ok_or(EIO)?.qos_pending = true; + producer = self.graph.append(cursors.producer, config_va.ok_or(EINVAL)?, + || packet.mark_published()) + .or_else(|error| { + self.pool.cancel_unpublished(target)?; + host.qos_cancel(qos)?; + self.active.back_mut().ok_or(EIO)?.qos_pending = false; + Err(error) + })?; + visible = true; + self.unpublished_binding = false; + // This Active and its Packet pins now own all visible records. + if let Some(configuration) = self.configuration.take() { configuration.release(); } + } else if first { host.register_queue(&crate::g17::fw::kick::KsmQueue::new( qid, kick_ring_va, @@ -2543,10 +2874,13 @@ impl Queue { } prepared_kick.commit(); self.submitted = self.submitted.checked_add(1).ok_or(EOVERFLOW)?; - let announce_va = self.graph.record(record_slot, ANNOUNCE, 0x40, announce)?; + let announce_va = if self.graph.m5 { + self.graph.record(record_slot, ANNOUNCE, 0x40, + crate::g17::fw::m5::queue::notification(qid, kick_timestamp)?)? + } else { self.graph.record(record_slot, ANNOUNCE, 0x40, announce)? }; producer = self.graph.append(producer, announce_va, || {})?; fence(Ordering::SeqCst); - if entry_signal { + if entry_signal && !self.graph.m5 { let predecessor_va = self.graph .record(record_slot, PREDECESSOR, 0x400, predecessor)?; @@ -2563,10 +2897,11 @@ impl Queue { self.graph.graph.gpu_va(), qid, producer.try_into()?, - self.outer_started & ring == 0, + activation.first, )?, )?; - self.outer_started |= ring; + let notify_activation = activation.first; + self.outer_started = activation.next_started; self.installed = true; self.published_config = Some(identity); if send_config { @@ -2602,6 +2937,13 @@ impl Queue { // ring itself and delayed dispatch. fence(Ordering::SeqCst); host.notify((0x83 << 48) | 0x02 | u64::from(policy.priority()) << 2)?; + if self.graph.m5 { + // The firmware work notification must precede a deferred host + // doorbell: an idle-policy hold alone does not wake gated cores. + host.submit_m5(&crate::g17::fw::m5::queue::HardwareSubmission::compute( + qid, self.kick.low_va(), policy.priority(), + first || notify_activation)?)?; + } crate::g17::trace::publish(crate::g17::trace::COMPUTE, qid, ordinal, &packet); host.note_submission() })(); @@ -2623,6 +2965,61 @@ impl Queue { Ok(()) } + /// The firmware consumes an empty channel installation before host-port + /// configuration. EAGAIN leaves the packet unpublished and is retried by + /// the common runtime with the device mutex released. + fn retire_configuration(&mut self, epoch: u64) -> Result { + let Some(configuration) = self.configuration.as_ref() else { return Ok(()); }; + if !configuration.consumed(self.graph.cursors()?, epoch) { return Ok(()); } + self.configuration.take().ok_or(EIO)?.release(); + if self.ordinal == 0 && self.active.len == 0 && self.replays.len == 0 { + self.retirement_ready = true; + if self.released { self.owner = None; } + } + Ok(()) + } + + fn install_m5(&mut self, packet: &Arc, host: &mut impl Host) -> Result { + let (epoch, state) = host.epoch()?; + if state == 0 { self.retire_configuration(epoch)?; } + if let Some(end) = self.m5_install { + return if self.graph.cursors()?.consumer == end || self.ordinal != 0 { Ok(()) } + else { Err(EAGAIN) }; + } + let context = &packet.context; + let policy = context.policy(); + let config = QueueConfig::new(&QueueConfigArgs { + target: QueueConfigTarget::Compute, kick_ring_va: self.kick.low_va(), + kick_ring_fw_va: self.kick.firmware_va(), qid: self.qid(), install: true, + completion_seed: None, context_id: context.id().try_into()?, + free_list: FreeListBinding { va: 0, generation: u64::MAX, slot: u32::MAX }, + scheduler_va: context.scheduler_va(), policy, qos_slot: context.qos_id(), + qos_update: true, owner_pid: context.owner_pid(), context_update: false, + })?; + if self.configuration.is_some() { return Err(EBUSY); } + // A queue reinstalled after a kill restarts at the firmware's consumer. + let start = self.graph.cursors()?.producer; + self.configuration = Some(configuration::Configuration::new( + context, (start + 1) % ITEM_RING_ENTRIES, epoch)?); + let result = (|| { + let va = self.graph.record(None, CONFIG, 0xc0, config)?; + let configuration = self.configuration.as_mut().ok_or(EIO)?; + let producer = self.graph.append(start, va, || configuration.mark_visible())?; + self.m5_install = Some(producer); + self.installed = true; + host.publish_qid(self.kick.id())?; + host.publish_outer(policy.priority(), &crate::g17::fw::channels::WorkSlot::new(DataMaster::Compute, + self.graph.graph.gpu_va(), self.qid(), producer.try_into()?, true)?)?; + self.outer_started = 1 << policy.priority(); + host.notify((0x83 << 48) | 0x02 | u64::from(policy.priority()) << 2)?; + Err(EAGAIN) + })(); + if result.is_err() && self.configuration.as_ref().is_some_and(|owner| !owner.visible()) { + self.configuration.take().ok_or(EIO)?.release(); + } + result + } + /// Everything that must hold before a publication changes any state: a healthy device /// epoch, room in the active ring (or this packet at the replay front), the bound /// owner's current context and a clean completion status. Returns the epoch and the @@ -2675,6 +3072,7 @@ impl Queue { &self, identity: &ConfigIdentity, first: bool, + priority_refresh: bool, context_id: u16, policy: crate::g17::fw::queue::Policy, packet: &Arc, @@ -2684,8 +3082,12 @@ impl Queue { kick_ring_va: self.kick.low_va(), kick_ring_fw_va: self.kick.firmware_va(), qid: self.qid(), - install: first, - completion_seed: None, + // An existing M5 queue needs the firmware install/update path to + // replace its private scheduling state when its class changes. + // The ordinary binding fields below are still processed afterward; + // no completion seed is supplied for a retained queue. + install: (first && !self.graph.m5) || priority_refresh, + completion_seed: (first && self.graph.m5).then_some(0), context_id, free_list: FreeListBinding { va: self.pool.control_va(), @@ -2790,6 +3192,7 @@ mod fault_ownership_tests { binding: Some(Lease::new(&used)), context: Lease::new(&used), publication, + moved: false, }); assert_eq!(used.load(Ordering::Relaxed), 2); Previous::retire_witnessed(&mut previous); @@ -2822,6 +3225,7 @@ mod fault_ownership_tests { binding: None, context: Lease::new(&used), publication: Some((9, 3)), + moved: false, }); Previous::retire_witnessed(&mut previous); assert_eq!(used.load(Ordering::Relaxed), 0); @@ -2936,7 +3340,12 @@ impl Queue { self.graph.set_owner(&previous.context)?; } if let Some(mut previous) = self.previous.take() { - if let Some(binding) = previous.binding.take() { + if previous.moved { + // As in `release_owner`: left detached, the new owner's views returned. + retired = self.binding.take(); + self.retained_owner = Some(previous.context.retain_firmware_pages()); + self.needs_fresh_backing = true; + } else if let Some(binding) = previous.binding.take() { retired = self.binding.replace(binding); } else { self.binding.as_mut().ok_or(EIO)?.context = previous.context; diff --git a/drivers/gpu/drm/asahi/g17/queue/compute/activation.rs b/drivers/gpu/drm/asahi/g17/queue/compute/activation.rs new file mode 100644 index 0000000000000..b306bbe820cc0 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/queue/compute/activation.rs @@ -0,0 +1,77 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! A retained M5 channel belongs to its current priority ring, not every ring it +//! has previously used. A priority change must submit it to the new ring again, +//! including when a later owner returns to an earlier priority. + +pub(super) struct Publication { + pub(super) first: bool, + /// Refresh firmware's per-QID scheduling state on a retained class change. + pub(super) refresh: bool, + pub(super) next_started: u8, +} + +impl Publication { + pub(super) fn new(started: u8, priority: u8, m5: bool) -> Self { + let ring = 1u8 << priority; + Self { + first: started & ring == 0, + refresh: m5 && started != 0 && started & ring == 0, + next_started: if m5 { ring } else { started | ring }, + } + } +} + +#[cfg(test)] +mod tests { + use super::Publication; + + #[test] + fn first_publication_activates() { + for priority in 0..4 { + let publication = Publication::new(0, priority, true); + assert!(publication.first); + assert!(!publication.refresh); + assert_eq!(publication.next_started, 1 << priority); + } + } + + #[test] + fn same_priority_keeps_fast_path() { + for priority in 0..4 { + let publication = Publication::new(1 << priority, priority, true); + assert!(!publication.first); + assert!(!publication.refresh); + assert_eq!(publication.next_started, 1 << priority); + } + } + + #[test] + fn returning_to_old_priority_reactivates() { + for old in 0..4 { + for new in 0..4 { + if old == new { continue; } + let changed = Publication::new(1 << old, new, true); + assert!(changed.first); + assert!(changed.refresh); + assert_eq!(changed.next_started, 1 << new); + let returned = Publication::new(changed.next_started, old, true); + assert!(returned.first); + assert!(returned.refresh); + assert_eq!(returned.next_started, 1 << old); + } + } + } + + #[test] + fn non_m5_preserves_historical_membership() { + let changed = Publication::new(1 << 2, 3, false); + assert!(changed.first); + assert!(!changed.refresh); + assert_eq!(changed.next_started, (1 << 2) | (1 << 3)); + let returned = Publication::new(changed.next_started, 2, false); + assert!(!returned.first); + assert!(!returned.refresh); + assert_eq!(returned.next_started, changed.next_started); + } +} diff --git a/drivers/gpu/drm/asahi/g17/queue/compute/configuration.rs b/drivers/gpu/drm/asahi/g17/queue/compute/configuration.rs new file mode 100644 index 0000000000000..7531ee2b0b33d --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/queue/compute/configuration.rs @@ -0,0 +1,40 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Initial M5 queue installation names physical execution/QoS identities +//! before an Active command exists. Retain the exact tokens independently. + +use super::{Context, Cursors, ITEM_RING_ENTRIES}; +use crate::mmu::ContextPin; +use kernel::{prelude::*, sync::Arc}; + +pub(super) struct Configuration { + context: Arc, + pin: ContextPin, + qos: u8, + target: u32, + epoch: u64, + visible: bool, +} +impl Configuration { + pub(super) fn new(context: &Arc, target: u32, epoch: u64) -> Result { + let pin = context.pin()?; + let qos = match context.pin_qos() { + Ok(qos) => qos, + Err(error) => { context.unpin(pin); return Err(error); } + }; + Ok(Self { context: context.clone(), pin, qos, target, epoch, visible: false }) + } + pub(super) fn mark_visible(&mut self) { self.visible = true; } + pub(super) fn visible(&self) -> bool { self.visible } + pub(super) fn consumed(&self, cursors: Cursors, epoch: u64) -> bool { + self.visible && self.epoch == epoch && self.target < ITEM_RING_ENTRIES + && cursors.count == ITEM_RING_ENTRIES + && cursors.producer == self.target && cursors.consumer == self.target + } + /// Only before visibility, after exact empty-channel consumption, after + /// transfer to the same pinned Active, or after verified processor stop. + pub(super) fn release(self) { + self.context.unpin_qos(self.qos); + self.context.unpin(self.pin); + } +} diff --git a/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs b/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs index 719a74787ca70..ace49e373311f 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs @@ -65,6 +65,9 @@ enum Phase { Published, Acknowledged, Unproven, + /// M5: the full-queue retirement witness covered the killed commands. Cached across + /// collector failures, since `submitted` still counts entries a retry already drained. + Witnessed, Released, } @@ -99,7 +102,7 @@ impl Queue { } pub(crate) fn exit_candidate(&self, context: &Arc, age: u64) -> bool { self.context().is_some_and(|bound| Arc::ptr_eq(bound, context)) - && self.released + && (self.released || self.timed_out()) && self.active.len != 0 && self.exit.is_none() && (!self.active_traversal() || self.fault_pending()) @@ -111,6 +114,35 @@ impl Queue { CANCELLED_DELAY_NS } } + /// M5: the watchdog failed this queue's command, and nothing has killed it yet. Its + /// commands may never retire on their own (a dispatch that does not terminate), so a kill + /// of its context treats it as failed work, like a command the firmware reported failed. + pub(crate) fn timed_out(&self) -> bool { + self.graph.m5 + && self.quarantined + && self.quarantine_error == Some(ETIMEDOUT) + && !self.spared_quarantine + && self.exit.is_none() + } + /// M5: a timed-out queue whose owner keeps it open, due for a kill of its context, with + /// how long ago the watchdog failed it. The owner's VM has already failed, so it loses + /// nothing it could still use; without the kill a command that never terminates keeps + /// running, and keeps this queue, until the owner closes. + pub(crate) fn open_timeout_kill(&self, now: u64) -> Option<(Arc, u64)> { + if self.released || self.timed_out_at == 0 || !self.timed_out() { + return None; + } + let context = self.context()?.clone(); + let age = now.saturating_sub(self.timed_out_at); + self.exit_candidate(&context, age).then_some((context, age)) + } + /// An open timed-out queue's kill is due within the exit polling window. + pub(crate) fn open_timeout_polling(&self, now: u64) -> bool { + !self.released + && self.timed_out_at != 0 + && self.timed_out() + && now.saturating_sub(self.timed_out_at) < POLL_NS + } /// A kill of `context` was published through this queue. pub(crate) fn exit_published(&self, context: &Arc) -> bool { self.exit.is_some() && self.context().is_some_and(|bound| Arc::ptr_eq(bound, context)) @@ -196,13 +228,13 @@ impl Queue { let Some(phase) = self.exit.as_ref().map(|exit| exit.phase) else { return Ok(()); }; - if !matches!(phase, Phase::Acknowledged | Phase::Unproven) { + if !matches!(phase, Phase::Acknowledged | Phase::Unproven | Phase::Witnessed) { return Ok(()); } // An unproven kill (acknowledged before the scheduler page showed the killed // state) is re-read on later passes and settles only once the page proves it, // even after its commands were retired by other means. - if self.active.len != 0 || phase == Phase::Unproven { + if (self.active.len != 0 || phase == Phase::Unproven) && phase != Phase::Witnessed { if !recovery_closed { return Ok(()); } @@ -217,7 +249,33 @@ impl Queue { self.exit.as_mut().ok_or(EIO)?.phase = Phase::Unproven; return Ok(()); } - if !self.active.iter().all(|active| { + if self.graph.m5 { + // A kill acknowledgement and the killed flag identify the control + // transaction, not the end of the firmware's reads, and M5 compute + // timestamp words stay zero even for executed work. The commands stay + // until the full-queue retirement witness covers the newest of them. + if self.active.len != 0 { + for active in self.active.iter() { + let Some(ticket) = active.ticket.as_ref() else { + return Ok(()); + }; + if self.valid_ticket(active, ticket).is_err() { + return Ok(()); + } + } + let newest = self.active.back().and_then(|active| active.ticket).ok_or(EIO)?; + if !super::retirement::quarantined_retired( + &newest, + self.graph.cursors()?, + self.witness_stamp()?, + self.active.len as u32, + self.submitted, + )? { + return Ok(()); + } + } + self.exit.as_mut().ok_or(EIO)?.phase = Phase::Witnessed; + } else if !self.active.iter().all(|active| { active .packet .completion diff --git a/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs b/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs index 6b1c49e6b4b73..8b3ab4b64dec1 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs @@ -141,7 +141,20 @@ pub(crate) fn note_stampless(qid: u8, kick: u64) { } } -/// A zero-duration record fails before visibility polling. It proves execution did not occur; +/// Firmware retires the compute commands in flight at a restart without executing them and +/// stores a record whose end is at most this many timer ticks after its start (0 or 1 tick +/// observed). The shortest executed command record seen is 18 ticks. +pub(crate) const CANCELLED_TICKS: u64 = 1; + +/// The record shows the command never executed: zero duration, or, for a command that was in +/// flight across a recovery (`crossed`), the restart's cancellation record. +pub(crate) fn unexecuted(record: Observation, crossed: bool) -> bool { + let [start, end] = record.timestamps; + start == end || (crossed && start != 0 && end >= start && end - start <= CANCELLED_TICKS) +} + +/// A record showing the command never executed ([`unexecuted`]) fails before visibility +/// polling. It proves execution did not occur; /// ownership still requires a separate retirement witness before mappings can be released. /// The command is complete when its stamp covers it, or when its exact completion record has /// been present for [`STAMP_FALLBACK_NS`] without one (`record_aged`) in the same recovery @@ -162,10 +175,11 @@ pub(crate) fn poll( next_end: Option, record_aged: bool, ) -> Result> { - if observed.is_some_and(|record| record.timestamps[0] == record.timestamps[1]) { + let crossed = matches!(recordless, Recordless::CrossedRecovery); + if observed.is_some_and(|record| unexecuted(record, crossed)) { return Err(ENODATA); } - let same_generation = !matches!(recordless, Recordless::CrossedRecovery); + let same_generation = !crossed; let completed = stamp_covers(stamp, ticket.kick) || (same_generation && observed.is_some() && record_aged); if !completed || !cursors.consumed(ticket.item_producer, ticket.item_count)? { diff --git a/drivers/gpu/drm/asahi/g17/queue/mod.rs b/drivers/gpu/drm/asahi/g17/queue/mod.rs index b7037a1c35f75..e3be25d0ceb20 100644 --- a/drivers/gpu/drm/asahi/g17/queue/mod.rs +++ b/drivers/gpu/drm/asahi/g17/queue/mod.rs @@ -149,10 +149,10 @@ impl Queue { }) } - fn ensure_compute(&mut self, scs_pages: u16) -> Result { + fn ensure_compute(&mut self, command: &Validated) -> Result { // Validate every request, including an existing scheduler. Physical // admission happens after its DRM dependencies, in backend publication. - self.backend.ensure_compute(scs_pages)?; + self.backend.ensure_compute(command)?; if self.compute.is_none() { self.compute = Some(Scheduler::new( &self.dev, @@ -218,17 +218,18 @@ impl Queue { self.prune_frontier(index); self.frontiers[index].reserve(1, GFP_KERNEL)?; if engine == Engine::Compute { - let Validated::Compute { scratch, .. } = &command else { - return Err(EINVAL); - }; - self.ensure_compute(scratch.pages())?; + self.ensure_compute(&command)?; } // Only convert inputs when no independent host-prefix wait could keep // the retained source bookkeeping queued behind an external signal. if engine == Engine::Compute { super::trace::inputs(self.context.id(), order.sequence, 0, inputs.len(), None); } - let sources = if engine == Engine::Compute + // Ring order completes a converted input only if a kick waits for its + // ring predecessor; without that, keep the input's completion wait. + let host_waits = self.backend.host_producer_waits(); + let sources = if !host_waits + && engine == Engine::Compute && !inputs.is_empty() && !destinations.has_outputs() && self.frontiers[0].iter().all(|(sequence, fences)| { @@ -298,8 +299,14 @@ impl Queue { &[] }; let count = frontier.len(); + // Only a command that copies or resets queries can observe timestamps the + // host has not yet published for its predecessors. + let query_access = match &command { + Validated::Compute { query_access, .. } => *query_access, + Validated::Render { .. } => true, + }; let timestamp_prefix = order.wait_through[index]; - let timestamps = if engine == Engine::Compute && timestamp_prefix.is_some() { + let timestamps = if engine == Engine::Compute && query_access && timestamp_prefix.is_some() { let frontier = self.frontiers[index].as_slice(); let end = frontier .partition_point(|(sequence, _)| Order::contains(timestamp_prefix, *sequence)); @@ -322,9 +329,17 @@ impl Queue { inputs.len() + count + timestamp_count + failure_count, GFP_KERNEL, )?; - let mut checked_inputs = - KVec::with_capacity(inputs.len() + timestamp_count + failure_count, GFP_KERNEL)?; - let mut firmware = KVec::with_capacity(count, GFP_KERNEL)?; + // Host-waited producers are checked on the host instead of in firmware. + let (host_count, firmware_count) = if host_waits { + (count, 0) + } else { + (0, count) + }; + let mut checked_inputs = KVec::with_capacity( + inputs.len() + host_count + timestamp_count + failure_count, + GFP_KERNEL, + )?; + let mut firmware = KVec::with_capacity(firmware_count, GFP_KERNEL)?; for fence in failures.into_iter().flatten() { checked_inputs.push(fence.clone(), GFP_KERNEL)?; dependencies.push(fence.clone(), GFP_KERNEL)?; @@ -345,8 +360,15 @@ impl Queue { if fences.status() > 0 { continue; } - dependencies.push(fences.ready.clone(), GFP_KERNEL)?; - firmware.push(fences.completed.clone(), GFP_KERNEL)?; + if host_waits { + // Keep other-engine producers on the host until completion, and + // propagate their errors before publishing the consumer. + dependencies.push(fences.completed.clone(), GFP_KERNEL)?; + checked_inputs.push(fences.completed.clone(), GFP_KERNEL)?; + } else { + dependencies.push(fences.ready.clone(), GFP_KERNEL)?; + firmware.push(fences.completed.clone(), GFP_KERNEL)?; + } } // Each command owns every input fence; independent engines may publish in // either order and must each consume an input's error status before running. @@ -419,7 +441,7 @@ impl Queue { let mut preceding = [0; 2]; let mut pending = KVec::with_capacity(1, GFP_KERNEL)?; while let Some(command) = parser.next()? { - let payload = self.proof.validate( + let mut payload = self.proof.validate( self.context.vm(), self.range.clone(), &self.window, @@ -430,13 +452,18 @@ impl Queue { Validated::Render { .. } => Engine::Render, Validated::Compute { .. } => Engine::Compute, }; - let order = Order::new( + if !self.backend.supports_engine(engine) { return Err(ENOTSUPP); } + self.backend.prepare_command(&mut payload)?; + let mut order = Order::new( base, preceding, command.barriers, command.payload.prior(), engine, )?; + if let Some((barriers, prior)) = command.payload.tiling_prefixes() { + order.narrow_tiling(&Order::new(base, preceding, barriers, prior, engine)?)?; + } preceding[engine.index()] += 1; let destinations = match &payload { Validated::Render { timestamps, .. } => Destinations::resolve(objects, timestamps)?, @@ -482,6 +509,9 @@ impl crate::queue::Queue for Queue { ) -> Result { let mut outputs = Outputs::new(&self.fences, syncs, input_count, &self.allocation_failure)?; let result = (|| { + if !self.backend.supports_submission() { + return Err(ENOTSUPP); + } if self.context.status().get() != 0 { return Err(EIO); } @@ -498,8 +528,12 @@ impl crate::queue::Queue for Queue { } } let mut guards = guards.into_iter(); + // Where a firmware wait reports completion before a weak stream tail's writes are + // visible, only a command that declares a visible completion is waited for there. + let visible_required = self.backend.firmware_sources_need_visible_completion(); let firmware_source = pending.len() == 1 - && matches!(&pending[0].0, Validated::Compute { .. }) + && matches!(&pending[0].0, Validated::Compute { visible_completion, .. } + if *visible_completion || !visible_required) && !pending[0].2.has_outputs(); let aggregate = outputs.publish(self.context.clone(), firmware_source)?; for (command, order, destinations) in pending { diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index c66b5cb199061..ca504ed87db91 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -2,6 +2,9 @@ //! Physical paired tiling and fragment queues and their retained command storage. +pub(in crate::g17) mod generation; +mod configuration; +pub(in crate::g17) mod m5_render_submission; mod observation; pub(crate) use observation::Observed; mod dependencies; @@ -38,14 +41,20 @@ use retirement::{Ticket, Tracker}; /// are built: the covered earlier renders (their fragment completions) and the /// covered compute completion. Which kick waits for the render prefix is the /// command's choice (`RenderPass::fragment_barriers`); compute always orders -/// the tiling kick, since compute output may feed vertex work. +/// the tiling kick, since compute output may feed vertex work. A stage-prefix +/// command (`RenderPass::stage_prefixes`) also names the earlier renders its +/// tiling half waits for (`render_tiling`, whole-render completions) and, when +/// its fragment half waits on later compute work, that compute completion. #[derive(Copy, Clone)] pub(crate) struct RenderDependencies { pub(crate) render: [KickDependency; 2], pub(crate) render_count: usize, + pub(crate) render_tiling: [KickDependency; 2], + pub(crate) render_tiling_count: usize, pub(crate) tiling: [KickDependency; 2], pub(crate) tiling_count: usize, pub(crate) compute: Option, + pub(crate) fragment_compute: Option, /// Fragment kicks of other pairs' overlapping same-VM renders. pub(crate) hazards: [KickDependency; crate::g17::hazard::PUBLICATION_BARRIERS], pub(crate) hazard_count: usize, @@ -55,9 +64,12 @@ impl RenderDependencies { pub(crate) const EMPTY: Self = Self { render: [KickDependency::ZERO; 2], render_count: 0, + render_tiling: [KickDependency::ZERO; 2], + render_tiling_count: 0, tiling: [KickDependency::ZERO; 2], tiling_count: 0, compute: None, + fragment_compute: None, hazards: [KickDependency::ZERO; crate::g17::hazard::PUBLICATION_BARRIERS], hazard_count: 0, }; @@ -76,12 +88,16 @@ pub(crate) struct RenderAddresses { pub(crate) struct PublicationLease { pub(crate) free_list_generation: u64, pub(crate) buffer_token: u64, + /// Hardware context for which the M5 token was selected. + pub(crate) buffer_context: Option, } /// Host services borrow the device-locked firmware state. Resource construction /// and command geometry preparation do not borrow this interface. pub(crate) trait Host { fn epoch(&self) -> Result<(u64, u32)>; + /// The host's accepted recovery generation (no device access). + fn generation(&self) -> u64; fn stamps(&self, qids: [u8; 2]) -> Result<[u32; 2]>; /// Publishes `owners` in order with one pass over the QoS table. fn qos_publish_pair( @@ -91,7 +107,7 @@ pub(crate) trait Host { policy: crate::g17::fw::queue::Policy, ) -> Result<[qos::Publication; 2]>; fn qos_cancel(&mut self, publication: qos::Publication) -> Result; - fn install_pair(&mut self, registration: &RenderRegistration) -> Result; + fn install_pair(&mut self, registration: &Registration) -> Result; fn publish_qid(&mut self, id: kick::Id) -> Result; /// The next publication number, shared with compute (timeout ordering). fn next_publication(&mut self) -> Result; @@ -103,6 +119,13 @@ pub(crate) trait Host { commit: impl FnOnce() -> Result, late_tiling: impl FnOnce(), ) -> Result<[u32; 2]>; + fn publish_configuration_pair( + &mut self, + priority: u8, + fragment: &WorkSlot, + tiling: &WorkSlot, + prepare: impl FnOnce(), + ) -> Result; fn activate_render_pool(&mut self, id: u16, generation: u64) -> Result; fn publish_retained_pair( &mut self, @@ -126,6 +149,9 @@ struct Prepared { packet: Arc, ordinal: u64, mcache: Option, + /// M5 kick register arrays and header flags, encoded with the commands. + register_arrays: Option<[[Option; 4]; 2]>, + kick_flags: Option, } struct Active { /// Publication number, shared with compute (crate::g17::timeout). @@ -150,9 +176,18 @@ struct Active { /// apart before it is republished; a single sample may straddle an update. const ABANDON_GRACE_NS: u64 = 2_000_000; +/// The firmware-specific host transaction; queue ownership remains shared. +pub(crate) enum Registration { + Neo(RenderRegistration), + M5(crate::g17::fw::m5::queue::HardwareSubmission), +} + /// Installed kick backing remains owned even when this pair has no logical /// owner. Commands retain their exact descriptor and allocation generations. pub(crate) struct Pair { + /// The firmware identity of an M5 render pool, pinned while this pair + /// uses the pool. G17P pools have fixed identities and need no pin. + pool_pin: Option, kick_aliases: Option<[Option; 2]>, memory: KBox, kicks: [kick::Queue; 2], @@ -169,6 +204,7 @@ pub(crate) struct Pair { deflake: abi::Deflake, descriptor_flags: [u32; 2], variant: &'static crate::g17::fw::Variant, + board: Option<&'static crate::g17::fw::m5::initdata::Board>, installed: [bool; 2], configs: [Option; 2], cursors: [u32; 2], @@ -186,6 +222,8 @@ pub(crate) struct Pair { /// Retired, never-executed groups awaiting republication on this pair, in /// their original order. Nothing else is prepared on the pair before them. replay: Tracker>, + /// Recovery generation of this pair's last executed retirement. + executed_generation: Option, /// When the oldest parked group was parked or the last one republished. replay_since: Option, /// Scheduler page of the context of the last published command: the @@ -197,6 +235,19 @@ pub(crate) struct Pair { /// Completion values of the two kicks of the last publication, kept with the configuration /// cache: what each queue holds once that publication completed. published_completions: Option<[u64; 2]>, + /// M5: the queues' firmware identity (logical context, scheduler page, + /// owner) changed since the last configuration firmware consumed. Only + /// this sets the M5 configuration's context and QoS update bits; + /// `context_update` stays the host witness that retires `previous` and + /// gates the completion seed. Cleared once a publication carried it. + identity_update: bool, + /// Owner of the identities named by an M5 cold-install configuration + /// pair, until firmware consumed it or an Active took them over. + configuration: Option, + /// Hardware context named by the configurations firmware last took. + configured_context: Option, + /// QoS slot named by those configurations. + configured_qos: Option, } impl Pair { #[inline(never)] @@ -221,7 +272,12 @@ impl Pair { deflake: abi::Deflake, descriptor_flags: [u32; 2], variant: &'static crate::g17::fw::Variant, + board: Option<&'static crate::g17::fw::m5::initdata::Board>, ) -> Result> { + let pool_pin = free_list.pin_id()?; + if alloc.uat.is_m5() { + free_list.ensure_backing(alloc, context.vm(), context.vm())?; + } if slot == 0 || usize::from(slot) >= queues::RENDER_SLOTS || owner == 0 @@ -238,14 +294,11 @@ impl Pair { } return Err(error); } - let Validated::Render { pass, .. } = &first.command else { - return Err(EINVAL); - }; let word = first.completion.work_state()?.word(); if word == u32::MAX { return Err(EINVAL); } - let layout = target::Layout::new(pass, clusters, variant.programs)?; + let layout = m5_render_submission::layout(&first.command, clusters, variant.programs)?; let storage = KBox::::new_uninit(GFP_KERNEL)?; let (tiling, ta_alias) = kick::Queue::render(alloc, ids[0], context.vm(), rings)?; let (fragment, fragment_alias) = kick::Queue::render(alloc, ids[1], context.vm(), rings)?; @@ -261,8 +314,10 @@ impl Pair { 0, word, variant, + board, )?; storage.write_init(kernel::try_init!(Self { + pool_pin, kick_aliases: Some([ta_alias, fragment_alias]), memory, kicks: [tiling, fragment], @@ -279,6 +334,7 @@ impl Pair { deflake, descriptor_flags, variant, + board, installed: [false; 2], configs: [None; 2], cursors: [0; 2], @@ -290,11 +346,16 @@ impl Pair { previous: None, displaced: None, replay: Tracker::new(), + executed_generation: None, replay_since: None, context_update: true, executed: 0, switch: None, published_completions: None, + identity_update: true, + configuration: None, + configured_context: None, + configured_qos: None, })) } pub(crate) fn qids(&self) -> [u8; 2] { @@ -318,15 +379,26 @@ impl Pair { pub(crate) fn buffer_id(&self) -> u8 { self.memory.manager.buffer_id() } + /// Whether the pair's M5 board takes the G17D forms. + fn g17d_forms(&self) -> bool { + self.board.is_some_and(|board| board.g17d_forms()) + } + /// The parameter-buffer state an M5 PB token is bound to. + pub(crate) fn m5_pb_state_va(&self) -> Option { + self.memory.graph.is_m5().then(|| self.memory.manager.state_va()) + } pub(crate) fn manager(&mut self) -> &mut buffer::Manager { &mut self.memory.manager } pub(crate) fn in_flight(&self) -> bool { - self.active.len() != 0 || self.replay.len() != 0 + self.active.len() != 0 || self.replay.len() != 0 || self.configuration.is_some() } - /// A replay or an abandonment grace period needs the worker to poll. + /// Configuration consumption, a replay or an abandonment grace period + /// needs the worker to poll. pub(crate) fn replay_polling(&self) -> bool { - self.replay.len() != 0 || self.active.iter().any(|a| a.abandoned_since.is_some()) + self.configuration.is_some() + || self.replay.len() != 0 + || self.active.iter().any(|a| a.abandoned_since.is_some()) } pub(crate) fn replay_stalled(&self, now: u64) -> bool { self.replay.len() != 0 && crate::g17::timeout::background_stalled(self.replay_since, now) @@ -379,6 +451,8 @@ impl Pair { pass, attachments, ray_query, + resources: programs, + plan, .. } = &packet.command else { @@ -387,13 +461,22 @@ impl Pair { if packet.completion.status().get() != 0 { return Err(EIO); } + // The TA mesh rendering buffer is bound only to kicks with an API task + // (object) stage. Resolve it before any ordinal state is claimed so an + // allocation failure fails this kick cleanly. + let mesh_storage = if programs.as_ref().is_some_and(|p| p.uses_task()) { + self.memory.mesh_storage_register(alloc)? + } else { + 0 + }; let ordinal = self.ordinal; ordinal.checked_add(1).ok_or(EOVERFLOW)?; self.memory.prefetch_descriptors(ordinal); let work = packet.completion.work_state()?; let node = work.node_va()?; let word = work.word(); - let layout = target::Layout::new(pass, self.clusters, self.variant.programs)?; + let layout = + m5_render_submission::layout(&packet.command, self.clusters, self.variant.programs)?; let deflake_layout = self.deflake; let owners = self.memory.owners(); self.memory @@ -414,10 +497,12 @@ impl Pair { self.memory.lifecycle }; let (scratch, metric) = self.memory.manager.scene_registers(scene)?; - let timestamp_offset = ordinal as usize % memory::RECORDS * 16; + let timestamp_offset = self.memory.timestamp_offset(ordinal); let timestamp = self.memory.timestamps.gpu_va() + timestamp_offset as u64; let resources = abi::RenderResources { context_id: packet.context.id().try_into()?, + // On M5 too: a directly owned execution root has one generation, which the + // render tail token names. context_generation: packet.context.generation(), pb_slot: self.buffer_id().into(), free_list_slot: self.free_list.id().into(), @@ -436,6 +521,7 @@ impl Pair { gtp_va: self.memory.target.gtp_va(&layout)?, deflake_va: deflake, ta_status_va: ta_status, + mesh_storage, fragment_status_va: fragment_status, fragment_status_fw_va: fragment_fw, aux_fb_va: auxiliary, @@ -454,14 +540,14 @@ impl Pair { ordinal, ta: abi::RenderStage { qid: qids[0], - descriptor_va: self.memory.graph.descriptor_client(0, ordinal), + descriptor_va: self.memory.graph.descriptor_client(0, ordinal)?, state_word: word, stamp_va: addresses.stamps[0][0], aux_stamp_va: addresses.stamps[0][1], }, fragment: abi::RenderStage { qid: qids[1], - descriptor_va: self.memory.graph.descriptor_client(1, ordinal), + descriptor_va: self.memory.graph.descriptor_client(1, ordinal)?, state_word: word, stamp_va: addresses.stamps[1][0], aux_stamp_va: addresses.stamps[1][1], @@ -476,24 +562,37 @@ impl Pair { true => attachments[1].as_slice(), false => &[][..], }; - let result = + let result = if self.memory.graph.is_m5() { self.memory - .write_descriptors(&args, attachments, packet.context.id().try_into()?); - let mcache = match result { - Ok(mcache) => mcache, + .write_m5_commands( + &args, + &layout, + programs.as_ref().and(*plan), + programs.as_ref().map_or(0, |p| p.mesh_class()), + ) + .map(|(arrays, flags)| (None, Some(arrays), Some(flags))) + } else { + self.memory + .write_descriptors(&args, attachments, packet.context.id().try_into()?) + .map(|mcache| (mcache, None, None)) + }; + let (mcache, register_arrays, kick_flags) = match result { + Ok(encoded) => encoded, Err(error) => { self.memory.graph.release(ordinal)?; - self.memory.generation(ordinal)?.ordinal = None; + self.memory.generation(ordinal)?.clear(); self.memory.target.cancel(ordinal); return Err(error); } }; - self.memory.timestamps.write(timestamp_offset, [0u64; 2])?; + self.memory.clear_timestamps(ordinal)?; self.memory.manager.claim_scene(ordinal, scene)?; self.prepared = Some(Prepared { packet: packet.clone(), ordinal, mcache, + register_arrays, + kick_flags, }); Ok(()) } @@ -503,28 +602,47 @@ impl Pair { /// with the fragment-barriers flag the resolved prefix orders the fragment kick too and the /// tiling kick preserves the TA prefix across physical pairs. Otherwise the render prefix /// orders the tiling kick. A compute dependency always orders the tiling kick. +/// The completion classes name, for M5 kicks, what each barrier waits for: +/// 1 the tiling encoder, 2 a full channel or compute completion, 3 the +/// fragment encoder; the last class is the implicit parent's. struct RenderBarriers { - tiling: [KickDependency; 3], + tiling: [KickDependency; 5], tiling_count: usize, - fragment: [KickDependency; 3 + crate::g17::hazard::PUBLICATION_BARRIERS], + fragment: [KickDependency; 4 + crate::g17::hazard::PUBLICATION_BARRIERS], fragment_count: usize, + tiling_types: [u8; 6], + fragment_types: [u8; 5 + crate::g17::hazard::PUBLICATION_BARRIERS], } fn render_barriers( fragment_stage: bool, + stage_prefixes: bool, dependencies: &RenderDependencies, own_tiling: KickDependency, + compute_class: u8, ) -> RenderBarriers { - let mut tiling_barriers = [KickDependency::ZERO; 3]; + let fragment_stage = fragment_stage || stage_prefixes; + let mut tiling_barriers = [KickDependency::ZERO; 5]; let mut tiling_count = 0; - let mut fragment_barriers = [own_tiling; 3 + crate::g17::hazard::PUBLICATION_BARRIERS]; + let mut fragment_barriers = [own_tiling; 4 + crate::g17::hazard::PUBLICATION_BARRIERS]; let mut fragment_count = 1; + let mut tiling_types = [1; 6]; + let mut fragment_types = [3; 5 + crate::g17::hazard::PUBLICATION_BARRIERS]; + fragment_types[0] = 1; + for dependency in &dependencies.render_tiling[..dependencies.render_tiling_count] { + tiling_barriers[tiling_count] = *dependency; + tiling_types[tiling_count] = 2; + tiling_count += 1; + } for dependency in dependencies.render() { if fragment_stage { fragment_barriers[fragment_count] = *dependency; fragment_count += 1; } else { tiling_barriers[tiling_count] = *dependency; + // A full channel barrier uses the tiling encoder's class 2. + // Class 3 belongs to the fragment encoder. + tiling_types[tiling_count] = 2; tiling_count += 1; } } @@ -541,8 +659,31 @@ fn render_barriers( } if let Some(compute) = dependencies.compute { tiling_barriers[tiling_count] = compute; + tiling_types[tiling_count] = compute_class; tiling_count += 1; } + if let Some(compute) = dependencies.fragment_compute { + fragment_barriers[fragment_count] = compute; + fragment_types[fragment_count] = compute_class; + fragment_count += 1; + } + if stage_prefixes { + // Only identical words of the same class are interchangeable. + let mut kept = 0; + for read in 0..tiling_count { + if (0..kept).any(|i| { + tiling_barriers[i].word() == tiling_barriers[read].word() + && tiling_types[i] == tiling_types[read] + }) { + continue; + } + tiling_barriers[kept] = tiling_barriers[read]; + tiling_types[kept] = tiling_types[read]; + kept += 1; + } + tiling_count = kept; + tiling_types[kept] = 1; + } // Only tile stores overlap: the tiling kick stays concurrent. for hazard in &dependencies.hazards[..dependencies.hazard_count] { fragment_barriers[fragment_count] = *hazard; @@ -553,6 +694,8 @@ fn render_barriers( tiling_count, fragment: fragment_barriers, fragment_count, + tiling_types, + fragment_types, } } @@ -572,12 +715,24 @@ impl Pair { channels::WorkSlot, kick::{KickArgs, KickQos}, queue::{ - self, FreeListBinding, QueueConfig, QueueConfigArgs, QueueConfigTarget, - TilingBinding, + FreeListBinding, QueueConfig, QueueConfigArgs, QueueConfigTarget, TilingBinding, }, }; use core::sync::atomic::{fence, Ordering}; let (epoch, mcache, policy) = self.publication_preflight(&packet, host)?; + let m5 = self.memory.graph.is_m5(); + let context_id = packet.context.id(); + let qos_slot = packet.context.qos_id(); + if m5 { + if self.configured_context.is_some_and(|configured| configured != context_id) + || self.configured_qos.is_some_and(|configured| configured != qos_slot) + { + self.identity_update = true; + } + if lease.buffer_context.is_some_and(|bound| bound != context_id) { + return Err(EBUSY); + } + } self.graph_started = true; let ordinal = self.ordinal; let next_ordinal = ordinal.checked_add(1).ok_or(EOVERFLOW)?; @@ -592,7 +747,13 @@ impl Pair { let kicks = self.kicks.each_ref().map(kick::Queue::timestamp); let parents = self.kicks.each_ref().map(kick::Queue::parent); let work = packet.completion.work_state()?; - let work_slot = work.render_slot(self.slot)?; + // A channel moves to this pair only after every publication made through it is + // visible on both of its rings (the stamps cover ordinal + 1). + let work_slot = work.render_channel(self.slot, |old, published| { + host.stamps(old).is_ok_and(|stamps| { + stamps.into_iter().all(|stamp| retirement::stamp_covers(stamp, published)) + }) + })?; self.memory.bind_work_slot(ordinal, qids, work_slot)?; let payload = work.channel_payload(self.slot, ordinal)?; let stamps = host.stamps(qids)?; @@ -626,16 +787,18 @@ impl Pair { data_master: DataMaster::Fragment as u8, }, ]; + // M5 carries the identity update bits only on a changed identity. + let identity_update = if m5 { self.identity_update } else { self.context_update }; let mut configs = [None; 2]; for stage in 0..2 { - configs[stage] = Some(QueueConfig::new(&QueueConfigArgs { + let args = QueueConfigArgs { target: if stage == 0 { - QueueConfigTarget::Tiling(TilingBinding { + QueueConfigTarget::Tiling(Some(TilingBinding { pb_va: self.memory.manager.state_va(), pb_token: lease.buffer_token, pb_slot: self.buffer_id().into(), fragment_qid: qids[1], - }) + })) } else { QueueConfigTarget::Fragment }, @@ -643,11 +806,20 @@ impl Pair { kick_ring_fw_va: self.kicks[stage].firmware_va(), qid: qids[stage], install: !self.installed[stage], - completion_seed: covered[stage].then(|| { - kicks[stage] - .prev() - .completion_value(payload.wrapping_sub(1)) - }), + completion_seed: (covered[stage] + && (!m5 || !self.installed[stage] || self.context_update)) + .then(|| { + // The predecessor's timestamp with the stage's scratch payload: the + // previous pass's on fragment; with the G17D forms 0 on tiling, as the + // G17D's tiling queues are seeded (its fragment checks read the paired + // fragment queue's payload, not this one). + let tiling_zero = stage == 0 && m5 && self.g17d_forms(); + kicks[stage].prev().completion_value(if tiling_zero { + 0 + } else { + payload.wrapping_sub(1) + }) + }), context_id: context.id().try_into()?, free_list: FreeListBinding { va: self.free_list.control_va(), @@ -657,12 +829,35 @@ impl Pair { scheduler_va: context.scheduler_va(), policy, qos_slot: qos, - qos_update: self.context_update, + qos_update: identity_update, owner_pid: context.owner_pid(), - context_update: self.context_update, - })?); + context_update: identity_update, + }; + configs[stage] = Some(if m5 { + crate::g17::fw::m5::queue::config( + &args, + crate::g17::fw::m5::queue::ConfigProfile { + soft_fault_enabled: true, + extended: false, + parameter_manager: (stage == 0).then(|| u32::from(self.buffer_id())), + }, + )? + } else { + QueueConfig::new(&args)? + }); } let configs = [configs[0].ok_or(EIO)?, configs[1].ok_or(EIO)?]; + if m5 && self.registered.is_none() { + if self.installed == [true; 2] { + // The cold-install configuration pair must be consumed first. + let cursors = self.memory.graph.cursors()?; + if cursors.iter().zip(self.cursors).any(|(c, end)| c.read != end) { + return Err(EBUSY); + } + } else { + return self.publish_m5_configuration(&packet, configs, kicks, policy, ring, host); + } + } let cache = if self.recovery_generation == epoch { self.configs } else { @@ -694,10 +889,7 @@ impl Pair { if cursors[0].write != self.cursors[0] || cursors[1].write != self.cursors[1] { return Err(EBUSY); } - let queues = qids.map(|qid| { - self.memory.graph.queues.gpu_va() - + u64::from(qid) * size_of::() as u64 - }); + let queues = qids.map(|qid| self.memory.graph.queue_va(qid)); let Validated::Render { pass, ray_query, .. } = &packet.command @@ -705,13 +897,39 @@ impl Pair { return Err(EINVAL); }; let own_tiling = KickDependency::new(qids[0], kicks[0]).ok_or(EINVAL)?; - let barriers = render_barriers(pass.fragment_barriers, dependencies, own_tiling); + let barriers = + render_barriers(pass.fragment_barriers, pass.stage_prefixes, dependencies, own_tiling, + self.board.map_or(2, crate::g17::fw::m5::compute::explicit_dependency)); let tiling_barriers = &barriers.tiling[..barriers.tiling_count]; + // With the G17D forms a fragment kick with history also names its parent's + // predecessor, in class 1, after the self-parent (classes [1, 3, 1] for a lone pass). + let ancestor = if m5 && self.g17d_forms() { + self.kicks[1].ancestor() + } else { + None + }; + let mut fragment_types = [0u8; 5 + crate::g17::hazard::PUBLICATION_BARRIERS]; + fragment_types[..barriers.fragment_count + 1] + .copy_from_slice(&barriers.fragment_types[..barriers.fragment_count + 1]); + fragment_types[barriers.fragment_count + 1] = 1; + let fragment_typed = barriers.fragment_count + 1 + usize::from(ancestor.is_some()); + let barrier_types = [ + &barriers.tiling_types[..barriers.tiling_count + 1], + &fragment_types[..fragment_typed], + ]; + let prepared = self.prepared.as_ref().ok_or(EIO)?; + let m5_kick = match (prepared.register_arrays, prepared.kick_flags) { + (Some(arrays), Some(flags)) if m5 => Some((arrays, flags)), + (None, None) if !m5 => None, + _ => return Err(EIO), + }; let args = |stage: usize, barriers, arrays, mcache| KickArgs { qid: qids[stage], timestamp: kicks[stage], parent: parents[stage], + ancestor: if stage == 1 { ancestor } else { None }, barriers, + dependency_types: m5.then_some(barrier_types[stage]), descriptor_va: self.memory.graph.descriptor_va(stage, ordinal), queue_va: queues[stage], qos: KickQos { @@ -723,36 +941,60 @@ impl Pair { }, }, mcache, - event_mask: abi::RENDER_KICK_EVENT_MASK, + // M5 tiling kicks also retain the last event of the mask. + event_mask: if m5 && stage == 0 { + [0, 0, 0, 1u64 << 62] + } else { + abi::RENDER_KICK_EVENT_MASK + }, register_arrays: arrays, - scs_enabled: ray_query[stage].scratch.enabled(), + scs_enabled: match m5_kick { + Some((_, flags)) => flags.scratch[stage], + None => ray_query[stage].scratch.enabled(), + }, + compute_matrix: m5_kick.is_some_and(|(_, flags)| flags.matrix[stage]), priority: policy.priority(), }; let ta_args = args( 0, tiling_barriers, - abi::TaDescriptor::register_bindings( - self.memory.graph.descriptor_client(0, ordinal), - ray_query[0].scratch.enabled(), - self.variant.programs, - )?, + match m5_kick { + Some((arrays, _)) => arrays[0], + None => abi::TaDescriptor::register_bindings( + self.memory.graph.descriptor_client(0, ordinal)?, + ray_query[0].scratch.enabled(), + self.variant.programs, + )?, + }, None, ); let fragment_args = args( 1, &barriers.fragment[..barriers.fragment_count], - abi::FragmentDescriptor::register_bindings( - self.memory.graph.descriptor_client(1, ordinal), - ray_query[1].scratch.enabled(), - self.variant.programs, - )?, + match m5_kick { + Some((arrays, _)) => arrays[1], + None => abi::FragmentDescriptor::register_bindings( + self.memory.graph.descriptor_client(1, ordinal)?, + ray_query[1].scratch.enabled(), + self.variant.programs, + )?, + }, mcache, ); - let registration = RenderRegistration::new( - (qids[1], self.kicks[1].low_va()), - (qids[0], self.kicks[0].low_va()), - policy.priority(), - )?; + let registration = if m5 { + Registration::M5(crate::g17::fw::m5::queue::HardwareSubmission::new( + (qids[1], self.kicks[1].low_va()), + (qids[0], self.kicks[0].low_va()), + policy.priority(), + true, + )?) + } else { + Registration::Neo(RenderRegistration::new( + (qids[1], self.kicks[1].low_va()), + (qids[0], self.kicks[0].low_va()), + policy.priority(), + )?) + }; let [tiling_kick, fragment_kick] = &mut self.kicks; let tiling_kick = tiling_kick.prepare_entry(&ta_args)?; let fragment_kick = fragment_kick.prepare_entry(&fragment_args)?; @@ -804,10 +1046,16 @@ impl Pair { windows[0].target().try_into()?, self.outer_started & ring == 0, )?; - let event_slot = memory.graph.support.word(abi::storage::EVENT_SLOTS + 4)?; - let next_event = event_slot.load(Ordering::Relaxed).wrapping_add(1); - let inner = memory.graph.support.word(0x28000)?; - let next_inner = (next_ordinal as u32).wrapping_mul(2); + // M5 graphs keep no G17P event and inner-producer words. + let legacy_state = if m5 { + None + } else { + let event_slot = memory.graph.support.word(abi::storage::EVENT_SLOTS + 4)?; + let next_event = event_slot.load(Ordering::Relaxed).wrapping_add(1); + let inner = memory.graph.support.word(0x28000)?; + let next_inner = (next_ordinal as u32).wrapping_mul(2); + Some((event_slot, next_event, inner, next_inner)) + }; let flist_submission = self.free_list.prepare_submission(2)?; let pm_submission = memory.manager.prepare_submission()?; let published = host.next_publication()?; @@ -825,7 +1073,11 @@ impl Pair { previous, }, kicks, - context_update: self.context_update, + // On M5 the cold-install configuration may already have + // cleared context_update: the first publication after a + // rebind still retires an unpublished `previous`. + context_update: self.context_update + || (m5 && self.previous.as_ref().is_some_and(|p| p.publication.is_none())), failed: false, drained_result: None, abandoned_since: None, @@ -834,9 +1086,25 @@ impl Pair { .map_err(|_| EBUSY)?; self.prepared = None; packet.mark_published(); - lanes[1].publish_prefix(); + work.channel_published(self.slot, qids, next_ordinal); + // The Active now retains this Packet and its identity and QoS pins; it + // replaces the configuration-only identity owner. + if let Some(configuration) = self.configuration.take() { + configuration.release(); + } + // M5 fills both lanes now; the host-port transaction precedes either + // producer, which publish_inner advances. + if m5 { + for lane in &lanes { + lane.prepare(); + } + } else { + lanes[1].publish_prefix(); + } fragment_kick.commit(); - lanes[1].publish_kick(); + if !m5 { + lanes[1].publish_kick(); + } tiling_kick.commit(); let mut qos_publications = [None; 2]; let result = (|| { @@ -861,7 +1129,15 @@ impl Pair { pm_submission.commit(); Ok(()) }, - || lanes[0].publish_tiling(event_slot, next_event, inner, next_inner), + || match legacy_state { + Some((event_slot, next_event, inner, next_inner)) => { + lanes[0].publish_tiling(event_slot, next_event, inner, next_inner) + } + None => { + lanes[1].publish_kick(); + lanes[0].publish_kick(); + } + }, ) })(); let outer_next = match result { @@ -904,6 +1180,13 @@ impl Pair { self.recovery_generation = epoch; self.installed = [true; 2]; self.published_completions = Some(kicks.map(|kick| kick.completion_value(payload))); + if m5 { + // Both configurations of this publication carried any update bits, + // so firmware applies them before this kick. + self.configured_context = Some(context_id); + self.configured_qos = Some(qos_slot); + self.identity_update = false; + } self.outer_started |= ring; if let Some(previous) = self.previous.as_mut().filter(|p| p.publication.is_none()) { previous.publication = Some(ordinal); @@ -924,6 +1207,90 @@ impl Pair { ) } + /// M5 cold install: the queues consume their initial configuration + /// records, published as one configuration pair, before the first kick + /// pair. Returns EBUSY so the command publishes once they are consumed. + fn publish_m5_configuration( + &mut self, + packet: &Arc, + configs: [crate::g17::fw::queue::QueueConfig; 2], + kicks: [KickTimestamp; 2], + policy: crate::g17::fw::queue::Policy, + ring: u8, + host: &mut impl Host, + ) -> Result { + use crate::g17::fw::queue::ITEM_RING_ENTRIES; + if self.configuration.is_some() { + return Err(EBUSY); + } + let ordinal = self.ordinal; + let qids = self.qids(); + let lanes = [ + publication::Lane::new( + &self.memory.graph, + 0, + ordinal, + self.cursors[0], + configs[0], + true, + kicks[0], + false, + )?, + publication::Lane::new( + &self.memory.graph, + 1, + ordinal, + self.cursors[1], + configs[1], + true, + kicks[1], + false, + )?, + ]; + let ends = self.cursors.map(|base| (base + 1) % ITEM_RING_ENTRIES); + let slot = |stage: usize| -> Result { + WorkSlot::new( + DataMaster::Compute, + self.memory.graph.queue_va(qids[stage]), + qids[stage], + ends[stage].try_into()?, + true, + ) + }; + let (tiling, fragment) = (slot(0)?, slot(1)?); + self.configuration = Some(configuration::Configuration::new( + &packet.context, + ends, + host.generation(), + )?); + let configuration = self.configuration.as_mut().ok_or(EIO)?; + let configured = + host.publish_configuration_pair(policy.priority(), &fragment, &tiling, || { + // Ownership exists before either item producer can become visible. + configuration.mark_visible(); + for lane in &lanes { + lane.publish_prefix(); + } + }); + if let Err(error) = configured { + if self.configuration.as_ref().is_some_and(|owner| !owner.visible()) { + self.configuration.take().ok_or(EIO)?.release(); + } + return Err(error); + } + self.cursors = ends; + self.installed = [true; 2]; + self.configured_context = Some(packet.context.id()); + self.configured_qos = Some(packet.context.qos_id()); + self.configs = [Some(configs[0]), Some(configs[1])]; + self.context_update = false; + self.identity_update = false; + self.outer_started |= ring; + packet.context.mark_published(); + host.notify_render(policy.priority())?; + Err(EBUSY) + } + /// Everything that must hold before a publication changes any state: no alias work /// pending, a healthy device epoch, room in the active ring, this slot's packet, the /// prepared descriptors for this ordinal, a clean completion status and a registration @@ -1040,6 +1407,9 @@ impl retirement::Memory for Observer<'_, H> { fn pass(&mut self, ordinal: u64) -> Result<[u64; 2]> { self.memory.pass(ordinal) } + fn stage_timestamps(&mut self, ordinal: u64) -> Result<[u64; 4]> { + self.memory.stage_timestamps(ordinal) + } } impl Pair { @@ -1086,14 +1456,16 @@ impl Pair { /// Latches when the front group executed and the firmware consumed its ring entries while /// `stamps` still lag it, and reports whether that has held for /// [`retirement::STAMP_FALLBACK_NS`] (`now`, host ns). The observation scan then retires it - /// on its pass record instead of waiting for stamps no later event will move. + /// on its pass record instead of waiting for stamps no later event will move. Not on M5, + /// whose pass record can precede the group's writes: as for M5 compute, only the stamps + /// return the group's buffers. pub(crate) fn note_stamp_wait( &mut self, stamps: [u32; 2], state: u32, now: u64, ) -> Result { - if self.quarantined || self.prepared.is_some() || state != 0 { + if self.quarantined || self.prepared.is_some() || state != 0 || self.memory.graph.is_m5() { return Ok(false); } let Some(front) = self.active.front_mut() else { @@ -1222,10 +1594,9 @@ impl Pair { /// Sample PM feedback only after this pair's exact retirement. Busy pairs /// keep their current capacity and let firmware request growth as needed. pub(crate) fn host_growth_target(&mut self, packet: &Packet) -> Result> { - let Validated::Render { pass, .. } = &packet.command else { - return Err(EINVAL); - }; - let required = target::Layout::new(pass, self.clusters, self.variant.programs)?.tvb_blocks; + let required = + m5_render_submission::layout(&packet.command, self.clusters, self.variant.programs)? + .tvb_blocks; self.memory .manager .note_required(required, crate::g17::now_ns()); @@ -1395,6 +1766,19 @@ impl Pair { self.retire_front(None, state, Some(observed), host, defer) } + /// The M5 cold-install configuration pair was consumed exactly: its + /// identity owner is released. + fn retire_configuration(&mut self, host: &impl Host) -> Result { + let Some(configuration) = self.configuration.as_ref() else { + return Ok(false); + }; + if !configuration.consumed(self.memory.graph.cursors()?, host.generation()) { + return Ok(false); + } + self.configuration.take().ok_or(EIO)?.release(); + Ok(true) + } + fn retire_front( &mut self, masks: Option<[u64; 2]>, @@ -1402,6 +1786,19 @@ impl Pair { observed: Option, host: &mut impl Host, defer: &mut impl FnMut(Deferred) -> Result, + ) -> Result { + let configuration_retired = state == 0 && self.retire_configuration(host)?; + let retired = self.retire_front_command(masks, state, observed, host, defer)?; + Ok(retired || configuration_retired) + } + + fn retire_front_command( + &mut self, + masks: Option<[u64; 2]>, + state: u32, + observed: Option, + host: &mut impl Host, + defer: &mut impl FnMut(Deferred) -> Result, ) -> Result { if self.quarantined || state != 0 || !self.selected(masks) { return Ok(false); @@ -1479,13 +1876,29 @@ impl Pair { ); defer(Deferred::Retired( front.packet.completion.clone(), - Ok(timestamps), + front.packet.check_firmware_producers().map(|()| timestamps), ))?; self.memory.retire(ordinal)?; self.active.pop().ok_or(EIO)?; + self.executed_generation = Some(host.generation()); Ok(true) } + /// An executed group of this pair retired in `generation` or later. + pub(crate) fn executed_since(&self, generation: u64) -> bool { + self.executed_generation.is_some_and(|executed| executed >= generation) + } + /// Published work of this pair is still live: in flight and not parked. + pub(crate) fn live_unparked(&self) -> bool { + self.replay.len() == 0 && self.active.len() != 0 + } + /// The oldest parked group precedes a command waiting through `through`. + pub(crate) fn replay_through(&self, through: Option) -> bool { + self.replay + .front() + .is_some_and(|packet| crate::g17::job::Order::contains(through, packet.order.sequence)) + } + /// Every retained group was retired and unlinked by firmware without /// executing (job list empty, stamps cover it, no recovery, fragment end /// never written): after another VM's recovery the firmware drops an @@ -2069,6 +2482,9 @@ impl Pair { /// Both processors have stopped before the caller reaches this method. /// VM ownership settles before quarantined admission is rehabilitated. pub(crate) fn stopped(&mut self, defer: &mut impl FnMut(Deferred) -> Result) -> Result { + if let Some(configuration) = self.configuration.take() { + configuration.release(); + } if let Some(prepared) = self.prepared.as_ref() { defer(Deferred::Unpublished( prepared.packet.completion.clone(), @@ -2150,9 +2566,7 @@ fn install_memory( descriptors.word(offset + 12)?, ]; let previous = words.map(|word| word.load(Ordering::Relaxed)); - let pages: u32 = (memory.manager.blocks() - * crate::hw::t8140::resources::TVB_PAGES_PER_BLOCK) - .try_into()?; + let pages = memory.manager.page_count()?; let next = crate::g17::fw::buffer::reload_descriptor( previous, memory.manager.page_list_va(), @@ -2192,6 +2606,7 @@ fn install_memory( /// A binding prepared while the registry exclusively lends the idle pair. /// Its frontier prevents a stale off-lock result from replacing another owner. pub(crate) struct PreparedBinding { + pool_pin: Option, previous_context: Arc, previous_owner: Option, ordinal: u64, @@ -2213,8 +2628,14 @@ pub(crate) struct RetiredBinding { buffer_id: u8, publication: Option, ready: bool, + /// An M5 same-context pool rebind: the old pool stays until firmware + /// acknowledged its release. + wait_for_pool_release: bool, } impl RetiredBinding { + pub(crate) fn wait_for_pool_release(&self) -> bool { + self.wait_for_pool_release + } pub(crate) fn context(&self) -> &Arc { &self.context } @@ -2232,6 +2653,7 @@ impl Pair { return; } self.context_update = false; + self.identity_update = false; if let Some(previous) = self.previous.as_mut() { if previous .publication @@ -2360,6 +2782,28 @@ impl Pair { && previous_ready && self.idle_for_context(self.context())?) } + /// An M5 pool may be replaced without changing the pair's live logical + /// owner. Commands already using the pair must drain, and an earlier + /// replacement must have retired, unless nothing was published since it: + /// that binding never reached firmware and is superseded directly. + pub(crate) fn bindable(&self, owner: u64, context: &Arc) -> Result { + if self.memory.graph.is_m5() + && self.owner == Some(owner) + && Arc::ptr_eq(context, self.context()) + { + return Ok(self.displaced.is_none() + && self.previous_supersedable() + && self.idle_for_context(context)?); + } + self.reusable() + } + /// No `previous`, or one that nothing has been published after: a warm + /// (memoryless) replacement with no publication ordinal and not ready. + fn previous_supersedable(&self) -> bool { + self.previous.as_ref().is_none_or(|p| { + p.memory.is_none() && p.publication.is_none() && !p.ready && self.context_update + }) + } /// Logical close leaves the physical channel identity and backing retained. pub(crate) fn release_owner(&mut self) { self.owner = None; @@ -2381,7 +2825,11 @@ impl Pair { pm_generation: Option, first: &Arc, ) -> Result { - if !self.reusable()? + let pool_pin = free_list.pin_id()?; + if alloc.uat.is_m5() { + free_list.ensure_backing(alloc, context.vm(), context.vm())?; + } + if !self.bindable(owner, &context)? || owner == 0 || !Arc::ptr_eq(&context, &first.context) || !context.is_current() @@ -2389,8 +2837,15 @@ impl Pair { return Err(EBUSY); } alloc.uat.vm_context_mask(context.vm())?; - let same_vm = Arc::ptr_eq(self.context().status(), context.status()); - let (memory, aliases, kicks) = if same_vm { + // On M5 only the same logical context (a pool rebind) keeps the graph: + // any other context gets its own graph and queue records, and a live + // queue record is never retargeted. + let keep_graph = if self.memory.graph.is_m5() { + Arc::ptr_eq(self.context(), &context) + } else { + Arc::ptr_eq(self.context().status(), context.status()) + }; + let (memory, aliases, kicks) = if keep_graph { if metrics.is_some() || buffer_id.is_some() || pm_generation.is_some() { return Err(EINVAL); } @@ -2399,10 +2854,8 @@ impl Pair { let metrics = metrics.ok_or(EINVAL)?; let buffer_id = buffer_id.ok_or(EINVAL)?; let pm_generation = pm_generation.ok_or(EINVAL)?; - let Validated::Render { pass, .. } = &first.command else { - return Err(EINVAL); - }; - let layout = target::Layout::new(pass, self.clusters, self.variant.programs)?; + let layout = + m5_render_submission::layout(&first.command, self.clusters, self.variant.programs)?; let aliases = [ self.kicks[0].prepare_client_alias(context.vm())?, self.kicks[1].prepare_client_alias(context.vm())?, @@ -2419,10 +2872,12 @@ impl Pair { self.ordinal, first.completion.work_state()?.word(), self.variant, + self.board, )?; (Some(memory), Some(persistent), Some(aliases)) }; Ok(PreparedBinding { + pool_pin, previous_context: self.context().clone(), previous_owner: self.owner, ordinal: self.ordinal, @@ -2441,7 +2896,7 @@ impl Pair { alloc: &Allocator<'_>, descriptors: &crate::g17::object::KernelObject, ) -> Result { - if !self.reusable()? + if !self.bindable(binding.owner, &binding.context)? || binding.ordinal != self.ordinal || binding.previous_owner != self.owner || !Arc::ptr_eq(&binding.previous_context, self.context()) @@ -2450,20 +2905,44 @@ impl Pair { return Err(EBUSY); } let foreign = binding.memory.is_some(); + let same_context = Arc::ptr_eq(self.context(), &binding.context); if let Some(memory) = binding.memory.as_mut() { - install_memory( + // A VM holds one graph's pool mappings per slot. On M5 a new + // context of the same VM gets its own graph: move the drained + // outgoing graph's mappings out first, and back if the install fails. + let same_vm = Arc::ptr_eq(self.context().status(), binding.context.status()); + let moved = if self.memory.graph.is_m5() && same_vm { + self.memory.take_pool_mappings() + } else { + None + }; + let installed = install_memory( alloc, descriptors, self.slot, memory, &mut binding.aliases, &mut binding.kicks, - )?; + ); + if let Err(error) = installed { + if let Some((slot, mappings)) = moved { + self.memory.put_pool_mappings(slot, mappings)?; + } + return Err(error); + } + drop(moved); } else { - self.memory.set_owner(self.qids(), &binding.context)?; - } + // The same logical context keeps its M5 queue records: their host + // words already name it, and a rewrite would store firmware-owned + // words back while the previous kick may still execute. + self.memory.set_owner(self.qids(), &binding.context, !same_context)?; + } + let wait_for_pool_release = self.memory.graph.is_m5() + && same_context + && !Arc::ptr_eq(&self.free_list, &binding.pool); let old_context = self.context().clone(); let old_buffer = self.buffer_id(); + self.pool_pin = binding.pool_pin.take(); let old_pool = core::mem::replace(&mut self.free_list, binding.pool.clone()); let old_memory = binding .memory @@ -2476,6 +2955,7 @@ impl Pair { buffer_id: old_buffer, publication: None, ready: foreign, + wait_for_pool_release, }; if foreign { self.displaced = Some(retired); @@ -2489,6 +2969,9 @@ impl Pair { self.graph_started = false; } self.context_update = true; + if foreign || !same_context { + self.identity_update = true; + } if foreign { self.cursors = [0; 2]; self.configs = [None; 2]; diff --git a/drivers/gpu/drm/asahi/g17/queue/render/configuration.rs b/drivers/gpu/drm/asahi/g17/queue/render/configuration.rs new file mode 100644 index 0000000000000..856a82b82a767 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/queue/render/configuration.rs @@ -0,0 +1,41 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! The initial M5 configuration can reach firmware before an Active command. +//! Its captured identities therefore have an independent owner. No aggregate +//! work-state node is referenced by this configuration-only item prefix. + +use super::{retirement::Cursors, Context}; +use crate::mmu::ContextPin; +use kernel::{prelude::*, sync::Arc}; + +pub(super) struct Configuration { + context: Arc, + pin: ContextPin, + qos: u8, + targets: [u32; 2], + generation: u64, + visible: bool, +} +impl Configuration { + pub(super) fn new(context: &Arc, targets: [u32; 2], generation: u64) -> Result { + let pin = context.pin()?; + let qos = match context.pin_qos() { + Ok(qos) => qos, + Err(error) => { context.unpin(pin); return Err(error); } + }; + Ok(Self { context: context.clone(), pin, qos, targets, generation, visible: false }) + } + pub(super) fn mark_visible(&mut self) { self.visible = true; } + pub(super) fn visible(&self) -> bool { self.visible } + pub(super) fn consumed(&self, cursors: [Cursors; 2], generation: u64) -> bool { + self.visible && self.generation == generation && cursors.iter().zip(self.targets).all(|(cursor, target)| { + cursor.done == target && cursor.read == target && cursor.write == target + }) + } + /// Only before visibility, after exact configuration consumption, after + /// transfer to an Active owning the same pinned Packet, or processor stop. + pub(super) fn release(self) { + self.context.unpin_qos(self.qos); + self.context.unpin(self.pin); + } +} diff --git a/drivers/gpu/drm/asahi/g17/queue/render/generation.rs b/drivers/gpu/drm/asahi/g17/queue/render/generation.rs new file mode 100644 index 0000000000000..d03857cc10a56 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/queue/render/generation.rs @@ -0,0 +1,174 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Per-command tiling, fragment-status and auxiliary storage. Mappings precede +//! their backing owners; a generation is reusable only after paired retirement. + +use crate::g17::{ + buffer::{self, BlockGeometry}, + fw::{ + m5::{initdata::Board, render_registers::ContextStore}, + render::{storage, RenderPass}, + }, + object::{Allocator, CpuMap, KernelObject, Placement}, +}; +use crate::{hw::t8140::{dynamic, resources as cfg}, mmu}; +use kernel::prelude::*; + +/// Offsets within a single client allocation, not fixed GPU addresses. +#[derive(Copy, Clone)] +pub(in crate::g17) struct Layout { + pub(in crate::g17) bytes: usize, + pub(in crate::g17) deflake: usize, + pub(in crate::g17) status: usize, + pub(in crate::g17) auxiliary: usize, + /// The viewport words the host writes before a pass, if it writes them. + pub(in crate::g17) viewport: Option, + pub(in crate::g17) viewport_control: u32, +} +impl Layout { + pub(in crate::g17) const NEO: Self = Self { + bytes: storage::STATE_SIZE, deflake: storage::DEFLAKE, + status: storage::TA_STATUS, auxiliary: storage::AUXILIARY, + viewport: Some(0x900), viewport_control: 0xc00, + }; + + /// M5: the context store at +0 ([`ContextStore`]), then the stage status page and the + /// auxiliary framebuffer, each on its own page above the store's extent. With the + /// two-cluster store the three end at 64 KiB; with the G17D's eight-cluster store at + /// 80 KiB. The host writes the viewport words into the state + /// stream, except with the G17D forms: there the stream is the hardware's, which overwrites it + /// when a pass is stored, and the host leaves the store alone. + const fn m5_for(store: ContextStore, host_viewport: bool) -> Self { + let page = mmu::UAT_PGSZ; + let status = store.extent.div_ceil(page) * page; + let auxiliary = status + page; + Self { + bytes: auxiliary + storage::AUXILIARY_SIZE, deflake: 0, status, auxiliary, + viewport: if host_viewport { Some(store.stream + 0x40) } else { None }, + viewport_control: 0x3000, + } + } + + /// `board`'s M5 layout. + pub(in crate::g17) const fn m5(board: &Board) -> Self { + Self::m5_for(ContextStore::selected(board), !board.g17d_forms()) + } + + pub(in crate::g17) fn prepare(self, state: &mut [u8], pass: &RenderPass) -> Result { + if state.len() < self.bytes { return Err(EINVAL); } + if let Some(viewport_offset) = self.viewport { + let viewport = storage::Viewport::new(pass.width, pass.height)? + .with_control(self.viewport_control); + let offset = self.deflake + viewport_offset; + let view = state.get_mut(offset..offset + size_of::()) + .ok_or(EINVAL)?.as_mut_ptr().cast::(); + if view.align_offset(align_of::()) != 0 { return Err(EINVAL); } + // SAFETY: The complete integer-only view is aligned and in bounds. + // The caller exclusively owns this unpublished command generation. + unsafe { viewport.apply(&mut *view) }; + } + state[self.auxiliary..self.auxiliary + storage::AUXILIARY_SIZE].fill(0); + Ok(()) + } +} + +const _: () = { + let two = Layout::m5_for(ContextStore::of(2), true); + assert!(two.bytes == 0x10000 && two.status == 0x4000 && two.auxiliary == 0x8000); + assert!(matches!(two.viewport, Some(0x9c0))); + let eight = Layout::m5_for(ContextStore::of(8), false); + assert!(eight.bytes == 0x14000 && eight.status == 0x8000 && eight.auxiliary == 0xc000); +}; + +enum Views { + Neo { + deflake: mmu::KernelMapping, + auxiliary: mmu::KernelMapping, + fragment_client: mmu::KernelMapping, + fragment: KernelObject, + }, + M5 { status_fw: mmu::KernelMapping, board: &'static Board }, +} + +pub(in crate::g17) struct Generation { + views: Views, + state: KernelObject, + pub(super) ordinal: Option, +} +impl Generation { + pub(in crate::g17) fn allocate_state(alloc: &Allocator<'_>, vm: &mmu::Vm, + board: Option<&'static Board>) -> Result { + let (range, layout) = if alloc.uat.is_m5() { + (BlockGeometry::M5_COMPACT, Layout::m5(board.ok_or(EINVAL)?)) + } else { (dynamic::CLIENT_LOWER, Layout::NEO) }; + KernelObject::new(alloc.dev, vm, Placement::In(range, cfg::TVB_PAGE), layout.bytes, + mmu::PROT_GPU_FW_SHARED_RW, CpuMap::WriteCombined) + } + + #[inline(never)] + pub(in crate::g17) fn new(alloc: &Allocator<'_>, vm: &mmu::Vm, state: KernelObject, + board: Option<&'static Board>) -> Result> { + let views = if alloc.uat.is_m5() { + let board = board.ok_or(EINVAL)?; + let layout = Layout::m5(board); + let status_fw = state.map_range(alloc.uat.kernel_vm(), + layout.status..layout.status + mmu::UAT_PGSZ, + alloc.uat.kernel_dynamic_range(), mmu::UAT_PGSZ as u64, + mmu::PROT_FW_SHARED_RW, 0)?; + Views::M5 { status_fw, board } + } else { + let deflake = buffer::compact_alias(&state, vm, + storage::DEFLAKE..storage::DEFLAKE + mmu::UAT_PGSZ, + mmu::UAT_PGSZ as u64, mmu::PROT_GPU_FW_SHARED_RW, mmu::UAT_PGSZ)?; + let auxiliary = state.map_range(vm, + storage::AUXILIARY..storage::AUXILIARY + storage::AUXILIARY_SIZE, + cfg::AUXILIARY, mmu::UAT_PGSZ as u64, mmu::PROT_GPU_FW_SHARED_RW, 0)?; + let fragment = alloc.kernel(mmu::UAT_PGSZ, mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_FW_SHARED_RW, CpuMap::WriteCombined)?; + let fragment_client = buffer::compact_alias(&fragment, vm, 0..mmu::UAT_PGSZ, + mmu::UAT_PGSZ as u64, mmu::PROT_GPU_SHARED_RW, mmu::UAT_PGSZ)?; + Views::Neo { deflake, auxiliary, fragment_client, fragment } + }; + Ok(KBox::new(Self { views, state, ordinal: None }, GFP_KERNEL)?) + } + + pub(in crate::g17) fn prepare(&mut self, ordinal: u64, pass: &RenderPass) -> Result { + if self.ordinal.is_some_and(|old| old != ordinal) { return Err(EBUSY); } + let layout = match &mut self.views { + Views::Neo { fragment, .. } => { + fragment.bytes_mut().fill(0); + Layout::NEO + } + Views::M5 { board, .. } => { + let layout = Layout::m5(board); + self.state.bytes_mut()[layout.status..layout.status + mmu::UAT_PGSZ].fill(0); + layout + } + }; + layout.prepare(self.state.bytes_mut(), pass)?; + self.ordinal = Some(ordinal); + Ok(()) + } + + pub(in crate::g17) fn addresses(&self) -> [u64; 5] { + match &self.views { + Views::Neo { deflake, auxiliary, fragment_client, fragment } => [ + deflake.iova(), self.state.gpu_va() + storage::TA_STATUS as u64, + fragment_client.iova(), fragment.gpu_va(), auxiliary.iova(), + ], + Views::M5 { status_fw, board } => { + let base = self.state.gpu_va(); + let layout = Layout::m5(board); + [base + layout.deflake as u64, base + layout.status as u64, + base + layout.status as u64 + 8, status_fw.iova() + 8, + base + layout.auxiliary as u64] + } + } + } + + pub(in crate::g17) fn retire(&mut self, ordinal: u64) -> Result { + if self.ordinal != Some(ordinal) { return Err(EIO); } + self.ordinal = None; + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/g17/queue/render/m5_render_submission.rs b/drivers/gpu/drm/asahi/g17/queue/render/m5_render_submission.rs new file mode 100644 index 0000000000000..333846d3f83c6 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/queue/render/m5_render_submission.rs @@ -0,0 +1,218 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! M5 render admission, target layout and command inputs on the shared submission path. + +use kernel::prelude::*; +use crate::g17::fw::m5::{render as cmd, render_registers as regs}; +use crate::g17::{buffer::target, command::Validated, fw::render::{RenderArgs, RenderPass, RegisterPrograms}}; + +/// Compressed single-layer MSAA ZLS admission: D32 depth with optional plain +/// stencil, and the compressed Z/S form. +pub(crate) fn depth_msaa_compression_enabled() -> bool { + crate::module_parameters::m5_depth_msaa_compression.value() == 1 +} + +pub(crate) fn validate(pass: &RenderPass) -> Result { + // The hardware carries the sample stride in eight bits. The 64-byte limit + // belonged to the first qualified profile, not the M5 imageblock. + let single_sample = pass.samples_log2 == 0 + && matches!((pass.utile_width, pass.utile_height), + (32, 32) | (32, 16) | (16, 16)) + && pass.sample_size % 8 == 0 + && u32::from(pass.sample_size) * u32::from(pass.utile_width) + * u32::from(pass.utile_height) <= 65536 + && (pass.sample_size != 0 || pass.layers == 1); + // ZLS addresses/actions and page strides are sample-independent. + // Retain the checked plain layout; compression is not admitted here, + // except in the layered compressed Z/S form (every enabled Z/S load or + // store compressed, a compression address per compressed plane, depth + // and stencil strides as supplied by userspace). The same form in + // single-layer 2x/4x MSAA depth/stencil passes is admitted only with the + // opt-in. + let plain_zls = pass.zls_control == 0 + || (pass.zls_control & 0x154 == 0 + && pass.depth.compression_va == 0 && pass.stencil.compression_va == 0); + let multisample = (plain_zls || layered_compressed_zls(pass) + || (depth_msaa_compression_enabled() && (single_layer_compressed_depth_zls(pass) + || single_layer_compressed_zls(pass)))) + && matches!(pass.samples_log2, 1..=3) + && matches!((pass.utile_width, pass.utile_height), + (32, 32) | (32, 16) | (16, 16)) + && pass.sample_size >= 8 && pass.sample_size % 8 == 0 + && (u32::from(pass.utile_width) * u32::from(pass.utile_height) + << pass.samples_log2) <= 2048 + && (u32::from(pass.sample_size) * u32::from(pass.utile_width) + * u32::from(pass.utile_height) << pass.samples_log2) <= 65536; + if !(single_sample || multisample) + || (pass.zls_control != 0 && pass.depth.va == 0 && pass.stencil.va == 0) { + return Err(EOPNOTSUPP); + } + Ok(()) +} + +/// Layered multisample depth/stencil in the compressed Z/S form. +fn layered_compressed_zls(pass: &RenderPass) -> bool { + pass.layers > 1 && compressed_zls_form(pass) +} + +/// Single-layer 2x/4x depth/stencil in the compressed Z/S form, the same form +/// as the layered one, compressed stencil included. +fn single_layer_compressed_zls(pass: &RenderPass) -> bool { + pass.layers == 1 && matches!(pass.samples_log2, 1 | 2) && compressed_zls_form(pass) +} + +/// Multisample depth/stencil in the compressed Z/S form: each enabled load or +/// store of a plane is compressed (ZLS bits: Z load 15 / compress 2, S load 14 +/// / compress 4, Z store 19 / compress 6, S store 18 / compress 8; the depth +/// format bits 25:26 are allowed), a plane +/// with any compression has a compression address, and no other ZLS bits. +fn compressed_zls_form(pass: &RenderPass) -> bool { + const Z_LOAD: u64 = 1 << 15; + const S_LOAD: u64 = 1 << 14; + const Z_STORE: u64 = 1 << 19; + const S_STORE: u64 = 1 << 18; + const Z_LOAD_COMPRESS: u64 = 1 << 2; + const S_LOAD_COMPRESS: u64 = 1 << 4; + const Z_STORE_COMPRESS: u64 = 1 << 6; + const S_STORE_COMPRESS: u64 = 1 << 8; + /// Depth format (bits 25:26) is a property of the plane, not of layering. + const Z_FORMAT: u64 = 3 << 25; + const KNOWN: u64 = Z_LOAD | S_LOAD | Z_STORE | S_STORE | Z_LOAD_COMPRESS | S_LOAD_COMPRESS + | Z_STORE_COMPRESS | S_STORE_COMPRESS | Z_FORMAT; + let control = pass.zls_control; + let z_compressed = control & (Z_LOAD_COMPRESS | Z_STORE_COMPRESS) != 0; + let s_compressed = control & (S_LOAD_COMPRESS | S_STORE_COMPRESS) != 0; + control != 0 + && control & !KNOWN == 0 + && (control & Z_LOAD == 0 || control & Z_LOAD_COMPRESS != 0) + && (control & S_LOAD == 0 || control & S_LOAD_COMPRESS != 0) + && (control & Z_STORE == 0 || control & Z_STORE_COMPRESS != 0) + && (control & S_STORE == 0 || control & S_STORE_COMPRESS != 0) + && (!z_compressed || (pass.depth.va != 0 && pass.depth.compression_va != 0)) + && (!s_compressed || (pass.stencil.va != 0 && pass.stencil.compression_va != 0)) + && (z_compressed || pass.depth.compression_va == 0) + && (s_compressed || pass.stencil.compression_va == 0) +} + +/// Single-layer D32 depth compression, with optional plain S8. +/// The compressed Z/S form is checked separately. Both depth compression bits +/// are required because partial rendering forces depth load and store even +/// when one of those operations is disabled in the final render pass. +fn single_layer_compressed_depth_zls(pass: &RenderPass) -> bool { + const Z_LOAD: u64 = 1 << 15; + const S_LOAD: u64 = 1 << 14; + const Z_STORE: u64 = 1 << 19; + const S_STORE: u64 = 1 << 18; + const Z_COMPRESS: u64 = (1 << 2) | (1 << 6); + const KNOWN: u64 = Z_LOAD | S_LOAD | Z_STORE | S_STORE | Z_COMPRESS; + pass.layers == 1 + && matches!(pass.samples_log2, 1 | 2) + && pass.zls_control & !KNOWN == 0 + && pass.zls_control & Z_COMPRESS == Z_COMPRESS + && pass.depth.va != 0 && pass.depth.compression_va != 0 + && pass.stencil.compression_va == 0 + && (pass.zls_control & (S_LOAD | S_STORE) == 0 || pass.stencil.va != 0) +} + +pub(crate) fn parameter_geometry(blocks: usize) -> Result { + let mut info = core::mem::MaybeUninit::::zeroed(); + // SAFETY: The complete sysinfo is initialized and writable. + unsafe { kernel::bindings::si_meminfo(info.as_mut_ptr()); } + // SAFETY: The object was initialized before si_meminfo filled it. + let info = unsafe { info.assume_init() }; + let ram = (info.totalram as u64).checked_mul(u64::from(info.mem_unit)).ok_or(EOVERFLOW)?; + let initial = blocks.checked_mul(0x80000).ok_or(EOVERFLOW)?; + crate::g17::buffer::BlockGeometry::m5(ram, initial as u64) +} + +/// The target layout of a render: G17P's for a command without an M5 plan. +pub(crate) fn layout(command: &Validated, clusters: u32, programs: RegisterPrograms) + -> Result { + let Validated::Render { pass, plan, .. } = command else { return Err(EINVAL); }; + let layout = target::Layout::new(pass, clusters, programs)?; + if plan.is_none() { return Ok(layout); } + validate(pass)?; + // One target group per cluster unless the pass asks for few primitives. + // Both target storage and the segment controls use the physical cluster count. + let geometry = parameter_geometry(layout.tvb_blocks)?; + // The auxiliary tables cover the complete parameter inventory, including + // future growth, rather than just the initially resident physical blocks. + let bytes = geometry.capacity.checked_mul(geometry.block_bytes).ok_or(EOVERFLOW)?; + let groups = if pass.few_primitives { 1 } else { clusters }; + layout.with_m5_slices(pass, clusters, groups, 64, bytes) +} + +pub(crate) fn controls( + pass: &RenderPass, + board: &'static crate::g17::fw::m5::initdata::Board, +) -> Result { + validate(pass)?; + let stage = |resource_flags| regs::M5RenderStageControls { + scs: crate::g17::render_resources::StageScratch::NONE, + resource_flags, resource_control: 0x200000001, ptc_max: 96, work_key: 0, + }; + Ok(regs::M5RenderControls { + board, + fragment_event_control: regs::fragment_event_control(true, 1), + ta: stage(0x4054b), fragment: stage(0x104054b), + ta_tile_control: 0x10c08860, ta_dispatch_control: 0x700, + fragment_raster_control: 0x200000ff0200, fragment_tile_flags: 0x200003717f, + store_pipeline_2_bind: pass.eot_rsrc_spec << 32, + store_pipeline_2: pass.usc_base + u64::from(pass.eot_offset & !7), + partial_store_pipeline_2_bind: pass.partial_eot_rsrc_spec << 32, + partial_store_pipeline_2: pass.usc_base + u64::from(pass.partial_eot_offset & !7), + partial_tile_flags: 0x200000, + aux_fb_flags: 0x10c000 | u64::from(pass.few_primitives) + | (u64::from(pass.depth_bias_is_int) << 18), + aux_fb_control: 0x100000, + }) +} + +pub(crate) fn inputs(args: &RenderArgs<'_>, usage: [u64; 2]) -> Result { + let r = args.resources; + let mut fields = cmd::M5RenderCommandInputs::default(); + let token = crate::g17::fw::m5::render_tail::context_token(u32::from(r.context_id), + u64::from(r.context_generation)).ok_or(EINVAL)?; + fields.ta.context_token = token; + fields.fragment.context_token = token; + fields.ta.resolved_tail_value = usage[0]; + fields.fragment.resolved_tail_value = Some(usage[1]); + fields.scheduler_record = r.kick_record_va; + // The compiler's external-write class of the render's graphics programs. + fields.ta.shader_write_class = cmd::shader_write_class(args.pass.shader_writes_memory); + fields.ta.sequence = args.ordinal.checked_mul(2).and_then(|n| n.checked_add(1)).ok_or(EOVERFLOW)?; + fields.fragment.sequence = args.ordinal.checked_mul(2).ok_or(EOVERFLOW)?; + fields.fragment.pass = 1; + fields.ta.predecessor_id = r.ta_ids.predecessor; + fields.fragment.predecessor_id = r.fragment_ids.predecessor; + fields.fragment.fragment_word_n = 1; + fields.fragment.fragment_flag_c = 1; + fields.fragment.binding_pairs[22][1] = 0x04040404; + // Pass policy is filled by apply_pass alongside its register copies. + fields.fragment.empty_tile_flags = [0; 4]; + fields.ta.timestamp_va = r.timestamp_va; + fields.ta.diagnostic_ta_timestamp_upper = r.timestamp_va + 8; + fields.fragment.timestamp_va = [r.timestamp_va + 16, r.timestamp_va + 24]; + let stamp = (args.ordinal as u32).wrapping_add(1).wrapping_shl(8); + fields.ta.linked_3d = cmd::M5TaLinked3d { + queue_index: u32::from(args.fragment.qid), stamp, ..Default::default() + }; + let completion = |stage: &crate::g17::fw::render::RenderStage, key| cmd::M5StageCompletion { + stamp_addresses: [stage.stamp_va, stage.aux_stamp_va], stamp, + queue_index: u32::from(stage.qid), ordinal: args.ordinal as u32, + key, ..Default::default() + }; + fields.ta.completion = completion(&args.ta, u64::from(r.ta_ids.current)); + fields.fragment.completion = completion(&args.fragment, u64::from(r.fragment_ids.current)); + fields.ta.pb_slot = r.pb_slot; + fields.ta.pb_state = r.pb_va; + fields.fragment.pm_state = r.pb_va; + fields.ta.pm_scene = r.scene_va; + fields.fragment.pm_scene = r.scene_va; + fields.ta.scene_end_va = r.scene_end_va; + fields.fragment.scene_end_va = r.scene_end_va; + fields.ta.uma_state = r.free_list_control_va; + fields.fragment.uma_state = r.free_list_control_va; + fields.ta.tail_flag = 1; + fields.fragment.tail_flag = 1; + Ok(fields) +} diff --git a/drivers/gpu/drm/asahi/g17/queue/render/memory.rs b/drivers/gpu/drm/asahi/g17/queue/render/memory.rs index 90a11ca977684..65615e1087037 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/memory.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/memory.rs @@ -19,6 +19,7 @@ use crate::{ }; use core::sync::atomic::Ordering; use kernel::{prelude::*, sync::Arc}; +use crate::g17::fw::m5::initdata::Board; pub(super) const RECORDS: usize = 32; pub(super) const TA_BYTES: usize = 0x14000; @@ -31,6 +32,9 @@ pub(super) const ANNOUNCE: [usize; 2] = [0x17000, 0x1b000]; pub(super) const JOB_LIST: usize = 0xa000; pub(super) const CONFIG: [usize; 2] = [0, 0x4000]; pub(super) const CONFIG_STRIDE: usize = 0x180; +/// Usage records of one M5 command, both stages. +const M5_USAGE_BYTES: usize = 0x40; +const M5_USAGE_PAIR_BYTES: usize = 2 * M5_USAGE_BYTES; /// The lead command of a stage (InitBM before tiling), after the stage's configuration record /// in its slot ([`crate::g17::fw::Variant::lead_commands`]). pub(super) const LEAD: usize = 0xc0; @@ -50,7 +54,7 @@ impl Aliases { vm: &mmu::Vm, source: core::ops::Range, ) -> Result { - let mut range = dynamic::LOWER; + let mut range = alloc.uat.kernel_lower_dynamic_range(); let size = (source.end - source.start) as u64; loop { // Descriptor aliases are read-only to GPU consumers; the canonical @@ -91,7 +95,8 @@ impl Aliases { /// Stable queue records and rotating descriptors/configuration records. pub(super) struct Graph { - aliases: [Aliases; 2], + /// One alias of all M5 command storage, or the G17P tiling and fragment ones. + aliases: [Option; 2], pub(super) descriptors: PooledObject, pub(super) queues: PooledObject, pub(super) config: PooledObject, @@ -99,8 +104,15 @@ pub(super) struct Graph { pub(super) owners: [Option; RECORDS], /// Host words of the pointer blocks in this firmware build. pub(super) pointers: fw::variant::PointerLayout, + m5: bool, } impl Graph { + pub(super) fn is_m5(&self) -> bool { self.m5 } + + /// Out of line: inlined, its temporaries stay reserved in `Memory::new`'s frame while the + /// rest of the pair (`buffer::Manager::new` and its allocations) is built below it, which + /// overflowed the submit ioctl's kernel stack. + #[inline(never)] pub(super) fn new( alloc: &Allocator<'_>, pool: &Arc, @@ -109,21 +121,30 @@ impl Graph { pointers: fw::variant::PointerLayout, ) -> Result { let prot = mmu::PROT_GPU_FW_SHARED_RW; + let m5 = alloc.uat.is_m5(); + // M5 selects cached firmware attributes at initial mapping creation; + // write-combined CPU stores also support the uncached capacity fallback. + let cpu = if m5 { CpuMap::WriteCombined } else { CpuMap::WriteBack }; + let bytes = if m5 { RECORDS * fw::m5::render_assembly::COMMAND_STORAGE_BYTES } + else { TA_BYTES + FRAGMENT_BYTES }; + // The GPU reads command storage only through the read-only aliases below. On M5 the + // kernel mapping, in the upper half every execution context includes, is firmware-only. let descriptors = pool.allocate( alloc, Family::Descriptor, - TA_BYTES + FRAGMENT_BYTES, - TA_BYTES + FRAGMENT_BYTES, - prot, - CpuMap::WriteBack, + bytes, + bytes, + if m5 { mmu::PROT_FW_SHARED_RW } else { prot }, + cpu, )?; - let ta = Aliases::new(&descriptors, alloc, context.vm(), 0..TA_BYTES)?; - let fragment = Aliases::new( + let ta = Aliases::new(&descriptors, alloc, context.vm(), + 0..if m5 { bytes } else { TA_BYTES })?; + let fragment = if m5 { None } else { Some(Aliases::new( &descriptors, alloc, context.vm(), TA_BYTES..TA_BYTES + FRAGMENT_BYTES, - )?; + )?) }; let mut queues = pool.allocate( alloc, Family::QueueState, @@ -138,27 +159,33 @@ impl Graph { 0x8000, 0x8000, prot, - CpuMap::WriteBack, + cpu, )?; let support = alloc.kernel( - storage::SUPPORT_SIZE, + if m5 { mmu::UAT_PGSZ } else { storage::SUPPORT_SIZE }, mmu::UAT_PGSZ as u64, prot, CpuMap::WriteCombined, )?; let address = queues.gpu_va(); for stage in 0..2 { - queues.write( - qids[stage] as usize * size_of::(), - queue::QueueRecord::new(&queue::QueueRecordArgs { - pointers_va: address + POINTERS[stage] as u64, - items_va: address + ITEMS[stage] as u64, - job_list_va: context.work_head_va(), - scheduler_va: context.scheduler_va(), - owner_pid: context.owner_pid(), - policy: context.policy(), - }), - )?; + let args = queue::QueueRecordArgs { + pointers_va: address + POINTERS[stage] as u64, + items_va: address + ITEMS[stage] as u64, + job_list_va: context.work_head_va(), + scheduler_va: context.scheduler_va(), + owner_pid: context.owner_pid(), + policy: context.policy(), + }; + if m5 { + queues.write(qids[stage] as usize * size_of::(), + fw::m5::queue::QueueRecord::new(&args)?)?; + } else { + queues.write( + qids[stage] as usize * size_of::(), + queue::QueueRecord::new(&args), + )?; + } queues.initialize::(POINTERS[stage], |view| { *view = queue::PointerBlock::new(&pointers)?; Ok(()) @@ -166,33 +193,67 @@ impl Graph { } queues.write(JOB_LIST, queue::JobListHead::new(address + JOB_LIST as u64))?; Ok(Self { - aliases: [ta, fragment], + aliases: [Some(ta), fragment], descriptors, queues, config, support, owners: [None; RECORDS], pointers, + m5, }) } - pub(super) fn offset(stage: usize, ordinal: u64) -> usize { + pub(super) fn queue_offset(&self, qid: u8) -> usize { + usize::from(qid) * if self.m5 { size_of::() } + else { size_of::() } + } + pub(super) fn queue_va(&self, qid: u8) -> u64 { + self.queues.gpu_va() + self.queue_offset(qid) as u64 + } + pub(super) fn offset(&self, stage: usize, ordinal: u64) -> usize { let slot = ordinal as usize % RECORDS; - if stage == 0 { + if self.m5 { + slot * fw::m5::render_assembly::COMMAND_STORAGE_BYTES + + stage * fw::m5::render_assembly::FRAGMENT_OFFSET + } else if stage == 0 { slot * abi::TA_DESCRIPTOR_SIZE } else { TA_BYTES + slot * abi::FRAGMENT_SLOT_SIZE } } pub(super) fn descriptor_va(&self, stage: usize, ordinal: u64) -> u64 { - self.descriptors.gpu_va() + Self::offset(stage, ordinal) as u64 + self.descriptors.gpu_va() + self.offset(stage, ordinal) as u64 } - pub(super) fn descriptor_client(&self, stage: usize, ordinal: u64) -> u64 { + pub(super) fn descriptor_client(&self, stage: usize, ordinal: u64) -> Result { + if self.m5 { + return Ok(self.aliases[0].as_ref().ok_or(EIO)?.client.iova() + + self.offset(stage, ordinal) as u64); + } let stride = if stage == 0 { abi::TA_DESCRIPTOR_SIZE } else { abi::FRAGMENT_SLOT_SIZE }; - self.aliases[stage].client.iova() + (ordinal as usize % RECORDS * stride) as u64 + Ok(self.aliases[stage].as_ref().ok_or(EIO)?.client.iova() + + (ordinal as usize % RECORDS * stride) as u64) + } + /// The command storage of `ordinal` on M5: both stages' commands. + pub(super) fn m5_storage(&mut self, ordinal: u64) -> Result<&mut [u8]> { + if !self.m5 || !self.owns(ordinal) { return Err(EINVAL); } + let start = self.offset(0, ordinal); + self.descriptors.bytes_mut() + .get_mut(start..start + fw::m5::render_assembly::COMMAND_STORAGE_BYTES) + .ok_or(EINVAL) + } + /// Reset the usage entries owned by this descriptor generation. They stay + /// resident with the graph until both stages release the descriptor slot. + pub(super) fn m5_usage(&mut self, ordinal: u64) -> Result<[u64; 2]> { + if !self.m5 || !self.owns(ordinal) { return Err(EINVAL); } + let start = ordinal as usize % RECORDS * M5_USAGE_PAIR_BYTES; + self.support.bytes_mut().get_mut(start..start + M5_USAGE_PAIR_BYTES) + .ok_or(EINVAL)?.fill(0); + let address = self.support.gpu_va() + start as u64; + Ok([address, address + M5_USAGE_BYTES as u64]) } pub(super) fn publish_recovery_targets(&self) -> Result { let Some(target) = self.pointers.recovery_target else { @@ -247,8 +308,24 @@ struct Compact { address: u64, _backing: KernelObject, } +/// The mesh storage chunk: chunk bases are 64 KiB aligned and the storage +/// register counts 64 KiB chunks. +const MESH_STORAGE_CHUNK: usize = 0x10000; + impl Compact { + /// M5 shared-RW object in the compact window with an explicit VA alignment. + fn new_aligned(alloc: &Allocator<'_>, vm: &mmu::Vm, size: usize, align: u64) -> Result { + let size = size.checked_add(mmu::UAT_PGSZ - 1).ok_or(EOVERFLOW)? & !(mmu::UAT_PGSZ - 1); + let backing = KernelObject::backing(alloc.dev, size, CpuMap::WriteCombined)?; + let mapping = backing.map_range(vm, 0..size, buffer::BlockGeometry::M5_COMPACT, + align, mmu::PROT_GPU_SHARED_RW, mmu::UAT_PGSZ)?; + if mapping.iova() % align != 0 { return Err(EINVAL); } + Ok(Self { address: mapping.iova(), mapping: Some(mapping), _backing: backing }) + } fn new(alloc: &Allocator<'_>, vm: &mmu::Vm, size: usize) -> Result { + if alloc.uat.is_m5() { + return Self::new_aligned(alloc, vm, size, cfg::TVB_PAGE); + } let backing = KernelObject::backing(alloc.dev, size, CpuMap::WriteCombined)?; let mapping = buffer::compact_alias( &backing, @@ -266,14 +343,65 @@ impl Compact { } } -pub(super) struct Generation { +/// Per-command state of a pair: the G17P one below or the M5 one +/// ([`super::generation::Generation`]). +pub(super) enum Generation { + Neo(KBox), + M5(KBox), +} +impl Generation { + fn allocate_state(alloc: &Allocator<'_>, vm: &mmu::Vm, board: Option<&'static Board>) + -> Result { + if alloc.uat.is_m5() { + super::generation::Generation::allocate_state(alloc, vm, board) + } else { + NeoGeneration::allocate_state(alloc, vm) + } + } + fn new(alloc: &Allocator<'_>, vm: &mmu::Vm, state: KernelObject, + board: Option<&'static Board>) -> Result { + Ok(if alloc.uat.is_m5() { + Self::M5(super::generation::Generation::new(alloc, vm, state, board)?) + } else { + Self::Neo(NeoGeneration::new(alloc, vm, state)?) + }) + } + pub(super) fn ordinal(&self) -> Option { + match self { + Self::Neo(generation) => generation.ordinal, + Self::M5(generation) => generation.ordinal, + } + } + /// Gives up the claim of an unpublished or retired command. + pub(super) fn clear(&mut self) { + match self { + Self::Neo(generation) => generation.ordinal = None, + Self::M5(generation) => generation.ordinal = None, + } + } + /// `viewport` applies to the G17P deflake buffer ([`abi::Deflake::viewport`]). + pub(super) fn prepare(&mut self, ordinal: u64, pass: &abi::RenderPass, viewport: bool) -> Result { + match self { + Self::Neo(generation) => generation.prepare(ordinal, pass, viewport), + Self::M5(generation) => generation.prepare(ordinal, pass), + } + } + pub(super) fn addresses(&self) -> [u64; 5] { + match self { + Self::Neo(generation) => generation.addresses(), + Self::M5(generation) => generation.addresses(), + } + } +} + +pub(super) struct NeoGeneration { deflake: mmu::KernelMapping, fragment_client: mmu::KernelMapping, state: KernelObject, fragment: KernelObject, pub(super) ordinal: Option, } -impl Generation { +impl NeoGeneration { fn allocate_state(alloc: &Allocator<'_>, vm: &mmu::Vm) -> Result { KernelObject::new( alloc.dev, @@ -368,21 +496,28 @@ impl Generation { } pub(super) struct Memory { - _fragment_view: mmu::KernelMapping, + _fragment_view: Option, pub(super) graph: Graph, scratch: Compact, discard: Compact, + /// M5 mesh rendering buffer of the queue's task-stage kicks. Shaders read + /// its per-workgroup 64 KiB chunk base, so it is mapped in the context VM. + mesh_storage: Option, + mesh_storage_chunks: u32, pub(super) target: KBox, - generations: [Option>; DEPTH], + generations: [Option; DEPTH], generation_count: usize, pub(super) timestamps: KernelObject, pub(super) manager: buffer::Manager, pub(super) lifecycle: [fw::ObjectIds; 2], // The extra client views retain their allocation positions even though // kick entries consume the register arrays embedded in the descriptors. - registers: KernelObject, + registers: Option, + m5_workspace: Option>, pub(super) context: Arc, pub(super) installed_pool: Option, + /// The M5 board ([`crate::g17::Shared::m5_board`]); `None` off M5. + pub(super) board: Option<&'static Board>, } impl Memory { #[inline(never)] @@ -402,16 +537,31 @@ impl Memory { ordinal: u64, word: u32, variant: &'static fw::Variant, + board: Option<&'static Board>, ) -> Result<(KBox, KVec)> { let storage = KBox::::new_uninit(GFP_KERNEL)?; let vm = context.vm(); + let m5 = alloc.uat.is_m5(); + if m5 != layout.is_m5() { return Err(EINVAL); } let mut graph = Graph::new(alloc, pool, &context, qids, variant.pointers)?; let mut target = target::Target::new(alloc.dev, vm, layout)?; target.prepare(ordinal, layout)?; - let state = Generation::allocate_state(alloc, vm)?; - let scratch = Compact::new(alloc, vm, cfg::SCENE_SIZE as usize)?; - let discard = Compact::new(alloc, vm, cfg::DISCARD_SIZE as usize)?; - let (tvb, mappings) = buffer::Tvb::new(alloc.dev, vm, layout.tvb_blocks)?; + let state = Generation::allocate_state(alloc, vm, board)?; + let (tvb, mappings, scratch, discard) = if m5 { + let geometry = super::m5_render_submission::parameter_geometry(layout.tvb_blocks)?; + let bytes = fw::m5::pool::parameter_scratch_stride(geometry.capacity, 0) + .map_err(|_| EINVAL)?.checked_mul(fw::m5::pool::SCENE_TABLE_ENTRIES) + .ok_or(EOVERFLOW)?; + let (tvb, mappings) = buffer::Tvb::with_geometry(alloc.dev, vm, layout.tvb_blocks, geometry)?; + let scratch = Compact::new(alloc, vm, bytes)?; + let discard = Compact::new(alloc, vm, 0x8000)?; + (tvb, mappings, scratch, discard) + } else { + let scratch = Compact::new(alloc, vm, cfg::SCENE_SIZE as usize)?; + let discard = Compact::new(alloc, vm, cfg::DISCARD_SIZE as usize)?; + let (tvb, mappings) = buffer::Tvb::new(alloc.dev, vm, layout.tvb_blocks)?; + (tvb, mappings, scratch, discard) + }; let count = DEPTH; let initial = if ordinal == 0 { 0 @@ -419,11 +569,11 @@ impl Memory { (ordinal ^ 1) as usize % count }; let mut generations = core::array::from_fn(|_| None); - generations[initial] = Some(Generation::new(alloc, vm, state)?); + generations[initial] = Some(Generation::new(alloc, vm, state, board)?); for offset in 1..count { let index = (initial + offset) % count; - let state = Generation::allocate_state(alloc, vm)?; - generations[index] = Some(Generation::new(alloc, vm, state)?); + let state = Generation::allocate_state(alloc, vm, board)?; + generations[index] = Some(Generation::new(alloc, vm, state, board)?); } let timestamps = alloc.kernel( mmu::UAT_PGSZ, @@ -447,36 +597,47 @@ impl Memory { let predecessor = crate::g17::ids::OBJECTS.object()?; let (fragment, tiling) = crate::g17::ids::OBJECTS.render_pair(predecessor)?; let support_va = graph.support.gpu_va(); - graph.support.initialize::(0, |view| { - view.init( - support_va, - alloc.uat.geometry().kernel_range().start + kernel_window::INDEX, - low::RENDER_RUN_LIST, - low::RENDER_PAGE_LIST, - word, - ) - })?; - let registers = KernelObject::new( + if !m5 { + graph.support.initialize::(0, |view| { + view.init( + support_va, + alloc.uat.geometry().kernel_range().start + kernel_window::INDEX, + low::RENDER_RUN_LIST, + low::RENDER_PAGE_LIST, + word, + ) + })?; + } + let registers = if m5 { None } else { Some(KernelObject::new( alloc.dev, vm, crate::g17::object::Placement::In(dynamic::CLIENT_LOWER, mmu::UAT_PGSZ as u64), mmu::UAT_PGSZ, mmu::PROT_GPU_SHARED_RO, CpuMap::WriteCombined, - )?; - let fragment_view = graph.descriptors.map_range( + )?) }; + let m5_workspace = if m5 { + let mut workspace = KBox::::new_uninit(GFP_KERNEL)?; + // SAFETY: Workspace is integer-only and has a valid zero representation. + unsafe { core::ptr::write_bytes(workspace.as_mut_ptr(), 0, 1); } + // SAFETY: The entire value was initialized above. + Some(unsafe { workspace.assume_init() }) + } else { None }; + let fragment_view = if m5 { None } else { Some(graph.descriptors.map_range( vm, TA_BYTES..TA_BYTES + mmu::UAT_PGSZ, dynamic::CLIENT_LOWER, mmu::UAT_PGSZ as u64, mmu::PROT_GPU_SHARED_RO, 0, - )?; + )?) }; Ok(( storage.write_init(kernel::try_init!(Self { graph, scratch, discard: discard, + mesh_storage: None, + mesh_storage_chunks: 0, target, generations, generation_count: count, @@ -485,35 +646,83 @@ impl Memory { lifecycle: [tiling, fragment], _fragment_view: fragment_view, registers: registers, + m5_workspace, context, installed_pool: None, + board, }))?, mappings, )) } + /// The mesh storage register value of a task-stage kick. The buffer is + /// allocated on the queue's first such kick and kept for the queue's lifetime, + /// so other contexts pay nothing. A failed allocation leaves no partial state. + /// A zero chunk parameter disables the binding. + pub(super) fn mesh_storage_register(&mut self, alloc: &Allocator<'_>) -> Result { + if self.mesh_storage.is_none() { + let chunks = crate::module_parameters::m5_mesh_storage_chunks.value(); + if chunks == 0 { return Ok(0); } + if chunks > 0xffff { return Err(EINVAL); } + let bytes = (chunks as usize).checked_mul(MESH_STORAGE_CHUNK).ok_or(EOVERFLOW)?; + let storage = Compact::new_aligned(alloc, self.context.vm(), bytes, + MESH_STORAGE_CHUNK as u64)?; + self.mesh_storage = Some(storage); + self.mesh_storage_chunks = chunks; + } + let storage = self.mesh_storage.as_ref().ok_or(ENOMEM)?; + Ok((storage.address & 0xffff_ffff_ffff) | (u64::from(self.mesh_storage_chunks) << 48)) + } pub(super) fn generation(&mut self, ordinal: u64) -> Result<&mut Generation> { self.generations[ordinal as usize % self.generation_count] - .as_deref_mut() + .as_mut() .ok_or(EIO) } pub(super) fn owns(&self, ordinal: u64) -> bool { self.graph.owns(ordinal) && self.generations[ordinal as usize % self.generation_count] .as_ref() - .is_some_and(|generation| generation.ordinal == Some(ordinal)) + .is_some_and(|generation| generation.ordinal() == Some(ordinal)) + } + /// M5 commands record four stage timestamps, G17P ones two. + pub(super) fn timestamp_offset(&self, ordinal: u64) -> usize { + ordinal as usize % RECORDS * if self.graph.m5 { 32 } else { 16 } + } + pub(super) fn clear_timestamps(&mut self, ordinal: u64) -> Result { + let at = self.timestamp_offset(ordinal); + if self.graph.m5 { self.timestamps.write(at, [0u64; 4]) } + else { self.timestamps.write(at, [0u64; 2]) } } pub(super) fn pass(&self, ordinal: u64) -> Result<[u64; 2]> { if !self.owns(ordinal) { return Err(EIO); } - let offset = ordinal as usize % RECORDS * 16; + let offset = self.timestamp_offset(ordinal); + Ok([ + self.timestamps.dword(offset)?.load(Ordering::Relaxed), + self.timestamps.dword(offset + if self.graph.m5 { 24 } else { 8 })? + .load(Ordering::Relaxed), + ]) + } + /// Export the four M5 stage words only after the observer has acquired + /// both completion stamps. Keep the two-word G17P representation intact. + pub(super) fn stage_timestamps(&self, ordinal: u64) -> Result<[u64; 4]> { + if !self.graph.m5 { + let [start, end] = self.pass(ordinal)?; + return Ok([start, start, start, end]); + } + if !self.owns(ordinal) { + return Err(EIO); + } + let offset = self.timestamp_offset(ordinal); Ok([ self.timestamps.dword(offset)?.load(Ordering::Relaxed), self.timestamps.dword(offset + 8)?.load(Ordering::Relaxed), + self.timestamps.dword(offset + 16)?.load(Ordering::Relaxed), + self.timestamps.dword(offset + 24)?.load(Ordering::Relaxed), ]) } pub(super) fn owners(&self) -> [Option; DEPTH] { - core::array::from_fn(|index| self.generations[index].as_ref().and_then(|g| g.ordinal)) + core::array::from_fn(|index| self.generations[index].as_ref().and_then(|g| g.ordinal())) } pub(super) fn retire(&mut self, ordinal: u64) -> Result { if !self.owns(ordinal) { @@ -522,7 +731,7 @@ impl Memory { self.target.retire(ordinal)?; self.manager.retire_scene(ordinal)?; self.graph.release(ordinal)?; - self.generation(ordinal)?.ordinal = None; + self.generation(ordinal)?.clear(); Ok(()) } } @@ -547,7 +756,7 @@ impl Memory { let record = ordinal as usize % RECORDS; let config = size_of::(); let ranges = [ - (&self.graph.descriptors, Graph::offset(0, ordinal), size_of::()), + (&self.graph.descriptors, self.graph.offset(0, ordinal), size_of::()), (&self.graph.config, CONFIG[0] + record * CONFIG_STRIDE, config), (&self.graph.config, CONFIG[1] + record * CONFIG_STRIDE, config), ]; @@ -566,6 +775,42 @@ impl Memory { } } + /// Encodes both M5 commands of `args.ordinal` into its command storage and + /// returns the kicks' register arrays and header flags. + pub(super) fn write_m5_commands( + &mut self, + args: &abi::RenderArgs<'_>, + layout: &target::Layout, + plan: Option, + mesh_class: u8, + ) -> Result<([[Option; 4]; 2], fw::kick::StageKickFlags)> { + use super::m5_render_submission as render_submission; + use fw::m5::{render_assembly, render_registers}; + if !self.graph.m5 { + return Err(EINVAL); + } + let mut controls = render_submission::controls(args.pass, self.board.ok_or(EINVAL)?)?; + if let Some(stages) = plan.and_then(|p| p.stages) { + controls.ta.scs = stages[0].scs; + controls.fragment.scs = stages[1].scs; + controls.ta.resource_flags = stages[0].resource_flags; + controls.ta.ptc_max = stages[0].ptc_max; + controls.fragment.resource_flags = stages[1].resource_flags; + controls.fragment.ptc_max = stages[1].ptc_max; + } + let usage = self.graph.m5_usage(args.ordinal)?; + let mut inputs = render_submission::inputs(args, usage)?; + inputs.ta.shifted_usage_hint = fw::m5::render::mesh_class_usage_hint(mesh_class); + render_registers::apply_pass(args, &controls, &mut inputs)?; + let extra = self.target.m5_extra(layout)?; + let low = self.graph.descriptor_client(0, args.ordinal)?; + let workspace = self.m5_workspace.as_deref_mut().ok_or(EIO)?; + let encoded = render_assembly::encode_into(self.graph.m5_storage(args.ordinal)?, low, + args, &controls, extra, render_registers::FragmentDispatch { words: [0; 2] }, + &inputs, workspace)?; + Ok((encoded.rce, encoded.kick_flags)) + } + pub(super) fn write_descriptors( &mut self, args: &abi::RenderArgs<'_>, @@ -576,7 +821,7 @@ impl Memory { .graph .descriptors .pointer( - Graph::offset(0, args.ordinal), + self.graph.offset(0, args.ordinal), size_of::(), )? .cast::(); @@ -584,7 +829,7 @@ impl Memory { .graph .descriptors .pointer( - Graph::offset(1, args.ordinal), + self.graph.offset(1, args.ordinal), size_of::(), )? .cast::(); @@ -626,15 +871,19 @@ impl Memory { if !self.owns(ordinal) { return Err(EIO); } + if self.graph.m5 { + return fw::m5::render_assembly::link_completions( + self.graph.m5_storage(ordinal)?, kicks, payload, fragment_busy); + } let tiling = self .graph .descriptors - .pointer(Graph::offset(0, ordinal), size_of::())? + .pointer(self.graph.offset(0, ordinal), size_of::())? .cast::(); let fragment = self .graph .descriptors - .pointer(Graph::offset(1, ordinal), size_of::())? + .pointer(self.graph.offset(1, ordinal), size_of::())? .cast::(); // SAFETY: The complete views were bounds/alignment checked during // preparation. The unchanged full ordinal still exclusively owns both @@ -655,7 +904,7 @@ impl Memory { let ta = self .graph .descriptors - .pointer(Graph::offset(0, ordinal), size_of::())? + .pointer(self.graph.offset(0, ordinal), size_of::())? .cast::(); // SAFETY: The view was checked during preparation and the ordinal still owns it; only // integer fields are read. @@ -677,21 +926,49 @@ impl Memory { impl Memory { /// Preserve the initial mapping order while moving VM-lifetime mappings /// out of the graph. Reserve all host storage before moving any mapping. + /// Moves this drained graph's pool mappings out of its VM slot, so a new + /// M5 graph of the same VM can install on the pair (see bind_owner). + pub(super) fn take_pool_mappings(&mut self) -> Option<(u8, mmu::RenderPoolMappings)> { + let slot = self.installed_pool?; + let id = u64::from(self.lifecycle[0].predecessor); + let mappings = self.context.vm().take_render_pool_mappings(slot, id)?; + self.installed_pool = None; + Some((slot, mappings)) + } + /// Returns mappings moved out by `take_pool_mappings` after a failed install. + pub(super) fn put_pool_mappings( + &mut self, + slot: u8, + mappings: mmu::RenderPoolMappings, + ) -> Result { + self.context.vm().put_render_pool_mappings(slot, mappings)?; + self.installed_pool = Some(slot); + Ok(()) + } + pub(super) fn take_persistent( &mut self, tvb: &mut KVec, ) -> Result> { if self.scratch.mapping.is_none() || self.discard.mapping.is_none() - || self.registers.gpu_va() == 0 + || (!self.graph.m5 && self.registers.as_ref().is_none_or(|r| r.gpu_va() == 0)) { return Err(EIO); } + if self.mesh_storage.as_ref().is_some_and(|m| m.mapping.is_none()) { + return Err(EIO); + } let mut aliases = KVec::new(); - aliases.reserve(tvb.len().checked_add(3).ok_or(EOVERFLOW)?, GFP_KERNEL)?; + aliases.reserve(tvb.len().checked_add(4).ok_or(EOVERFLOW)?, GFP_KERNEL)?; aliases.push(self.scratch.mapping.take().ok_or(EIO)?, GFP_KERNEL)?; aliases.push(self.discard.mapping.take().ok_or(EIO)?, GFP_KERNEL)?; - aliases.push(self.registers.take_mapping()?, GFP_KERNEL)?; + if let Some(storage) = &mut self.mesh_storage { + aliases.push(storage.mapping.take().ok_or(EIO)?, GFP_KERNEL)?; + } + if let Some(registers) = &mut self.registers { + aliases.push(registers.take_mapping()?, GFP_KERNEL)?; + } for mapping in core::mem::take(tvb) { aliases.push(mapping, GFP_KERNEL)?; } @@ -704,6 +981,9 @@ impl Memory { if slot >= 4 || !self.owns(ordinal) { return Err(EINVAL); } + if self.graph.m5 { + return fw::m5::render_assembly::bind_work_slot(self.graph.m5_storage(ordinal)?, slot); + } let queues = [ self.graph .queues @@ -715,12 +995,12 @@ impl Memory { let ta = self .graph .descriptors - .pointer(Graph::offset(0, ordinal), size_of::())? + .pointer(self.graph.offset(0, ordinal), size_of::())? .cast::(); let fragment = self .graph .descriptors - .pointer(Graph::offset(1, ordinal), size_of::())? + .pointer(self.graph.offset(1, ordinal), size_of::())? .cast::(); for word in queues { word.store(slot, Ordering::Relaxed); @@ -738,7 +1018,40 @@ impl Memory { /// Retargets both retired queue records to `context`. A context of another /// scheduling profile also replaces the priority and policy words, after /// the owner fields; the same profile leaves them untouched. - pub(super) fn set_owner(&self, qids: [u8; 2], context: &Context) -> Result { + /// `rewrite`: store the M5 records even when their host-owned words + /// already name `context`; without it that no-op read-modify-write, which + /// would also store back firmware-owned words of a possibly running queue, + /// is skipped. G17P always retargets. + pub(super) fn set_owner(&self, qids: [u8; 2], context: &Context, rewrite: bool) -> Result { + if self.graph.m5 { + let records = qids.map(|qid| self.graph.queues.pointer(self.graph.queue_offset(qid), + size_of::())); + let [first, second] = records; + let pointers = [first?, second?].map(|p| p.cast::()); + // SAFETY: Both complete records are in bounds. The caller owns + // retired queues exclusively; build both updates before storing. + unsafe { + let mut values = pointers.map(|p| p.read_unaligned()); + if !rewrite { + let mut named = true; + for value in &values { + named &= value.names_owner(context.work_head_va(), context.scheduler_va(), + context.owner_pid(), context.policy())?; + } + if named { + return Ok(()); + } + } + for value in &mut values { + value.set_compute_owner(context.work_head_va(), context.scheduler_va(), + context.owner_pid(), context.policy())?; + } + for (pointer, value) in pointers.into_iter().zip(values) { + pointer.write_unaligned(value); + } + } + return Ok(()); + } let offsets = qids.map(|qid| usize::from(qid) * size_of::()); let base = self.graph.queues.pointer(0, GRAPH_SIZE)?; // Check every field before the first update, retaining the required diff --git a/drivers/gpu/drm/asahi/g17/queue/render/parked.rs b/drivers/gpu/drm/asahi/g17/queue/render/parked.rs index 81d4306eda28c..83a4bcb2f3022 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/parked.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/parked.rs @@ -2,10 +2,9 @@ //! Installed kick storage retained after a logical scheduler release is acknowledged. -use super::{install_memory, memory, target, Pair, RetiredBinding, Tracker}; +use super::{install_memory, memory, Pair, RetiredBinding, Tracker}; use crate::g17::{ buffer, - command::Validated, context::Context, freelist::RenderPool, job::Packet, @@ -20,7 +19,7 @@ use kernel::{prelude::*, sync::Arc}; /// context awaiting an executed replacement keeps its separate retirement owner. pub(crate) struct ParkedPair { kicks: Option<[kick::Queue; 2]>, - _scheduler: Arc, + _scheduler: Arc, ids: [kick::Id; 2], slot: u8, ordinal: u64, @@ -28,6 +27,7 @@ pub(crate) struct ParkedPair { deflake: crate::g17::fw::render::Deflake, descriptor_flags: [u32; 2], variant: &'static crate::g17::fw::Variant, + board: Option<&'static crate::g17::fw::m5::initdata::Board>, installed: [bool; 2], recovery_generation: u64, registered: Option, @@ -38,10 +38,13 @@ pub(crate) struct ParkedPair { /// owner) keeps it, so its next owner change still waits while that /// scheduler is released. executed: u64, + /// The VM of the context that last ran on these rings. + vm: u64, } /// All new graph ownership stays separate from the installed kick storage until commit. pub(crate) struct Prepared { + pool_pin: Option, memory: Option>, pool: Arc, aliases: Option>, @@ -83,6 +86,7 @@ impl Pair { /// under the same device mutex. Moving these owners performs no firmware operation. pub(crate) fn park(self) -> (ParkedPair, RetiredBinding) { let context = self.memory.context.clone(); + let vm = context.vm().id(); let scheduler = context.scheduler_owner(); let buffer_id = self.buffer_id(); let parked = ParkedPair { @@ -95,12 +99,14 @@ impl Pair { deflake: self.deflake, descriptor_flags: self.descriptor_flags, variant: self.variant, + board: self.board, installed: self.installed, recovery_generation: self.recovery_generation, registered: self.registered, outer_started: self.outer_started, previous: self.previous, executed: self.executed, + vm, }; let retired = RetiredBinding { memory: Some(self.memory), @@ -109,12 +115,27 @@ impl Pair { buffer_id, publication: None, ready: true, + wait_for_pool_release: false, }; (parked, retired) } } impl ParkedPair { + /// Clears both kick rings when the pair goes to a context of another VM than the one + /// that last ran on it, so the new owner cannot read the entries the previous owner's + /// commands left. The pair is parked, so the firmware consumed every entry. + pub(crate) fn clear_for_vm(&mut self, vm: u64) -> Result { + if self.vm == vm { + return Ok(()); + } + for queue in self.kicks.as_mut().ok_or(EBUSY)?.iter_mut() { + queue.clear_render_ring()?; + } + self.vm = vm; + Ok(()) + } + /// The release of `scheduler` has been consumed: a later owner change no /// longer has to wait for it. pub(crate) fn release_consumed(&mut self, scheduler: u64) { @@ -143,19 +164,24 @@ impl ParkedPair { pm_generation: u64, first: &Arc, ) -> Result { + let pool_pin = free_list.pin_id()?; + if alloc.uat.is_m5() { + free_list.ensure_backing(alloc, context.vm(), context.vm())?; + } if owner == 0 || !Arc::ptr_eq(&context, &first.context) || !context.is_current() { return Err(EINVAL); } let kicks = self.kicks.as_ref().ok_or(EBUSY)?; - let Validated::Render { pass, .. } = &first.command else { - return Err(EINVAL); - }; let word = first.completion.work_state()?.word(); if word == u32::MAX { return Err(EINVAL); } alloc.uat.vm_context_mask(context.vm())?; - let layout = target::Layout::new(pass, self.clusters, self.variant.programs)?; + let layout = super::m5_render_submission::layout( + &first.command, + self.clusters, + self.variant.programs, + )?; let aliases = [ kicks[0].prepare_client_alias(context.vm())?, kicks[1].prepare_client_alias(context.vm())?, @@ -172,8 +198,10 @@ impl ParkedPair { self.ordinal, word, self.variant, + self.board, )?; Ok(Prepared { + pool_pin, memory: Some(memory), pool: free_list, aliases: Some(persistent), @@ -221,6 +249,7 @@ impl ParkedPair { return Err(EIO); }; storage.write_init(kernel::try_init!(Pair { + pool_pin: next.pool_pin.take(), graph_started: false, kick_aliases: None, memory, @@ -237,6 +266,7 @@ impl ParkedPair { deflake: self.deflake, descriptor_flags: self.descriptor_flags, variant: self.variant, + board: self.board, installed: self.installed, configs: [None; 2], cursors: [0; 2], @@ -248,11 +278,16 @@ impl ParkedPair { previous: self.previous.take(), displaced: None, replay: Tracker::new(), + executed_generation: None, replay_since: None, context_update: true, executed: self.executed, switch: None, published_completions: None, + identity_update: true, + configuration: None, + configured_context: None, + configured_qos: None, })) } } diff --git a/drivers/gpu/drm/asahi/g17/queue/render/publication.rs b/drivers/gpu/drm/asahi/g17/queue/render/publication.rs index d65c40c7cfade..838d3b17bab86 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/publication.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/publication.rs @@ -5,6 +5,7 @@ use super::{memory, retirement::Window}; use crate::g17::fw::{ + m5, kick::KickTimestamp, queue::{DataMaster, KickAnnounce, QueueConfig, ITEM_RING_ENTRIES}, }; @@ -27,6 +28,12 @@ pub(in crate::g17::queue) fn required(previous: Option, config: Que || previous.map(identity) != Some(identity(config)) } +/// The kick announcement record: the G17P one, or the M5 notification. +enum Announcement { + Neo(KickAnnounce), + M5(m5::queue::Notification), +} + pub(super) struct Lane<'a> { items: [&'a AtomicU64; 4], producer: &'a AtomicU32, @@ -34,8 +41,8 @@ pub(super) struct Lane<'a> { config: QueueConfig, config_destination: *mut u64, config_present: bool, - announce: KickAnnounce, - announce_destination: *mut KickAnnounce, + announce: Announcement, + announce_destination: *mut u8, prefix: usize, prefix_end: u32, pub(super) window: Window, @@ -55,7 +62,9 @@ impl<'a> Lane<'a> { timestamp: KickTimestamp, lead: bool, ) -> Result { - let count = 2 + u32::from(present) + u32::from(lead); + let m5 = graph.is_m5(); + // M5 items carry no descriptor: [configuration], notification. + let count = u32::from(present) + u32::from(lead) + if m5 { 1 } else { 2 }; let window = Window::new(base, count, ITEM_RING_ENTRIES)?; let offset = ordinal as usize % memory::RECORDS; let config_offset = memory::CONFIG[stage] + offset * memory::CONFIG_STRIDE; @@ -69,8 +78,8 @@ impl<'a> Lane<'a> { } let announce_destination = graph .queues - .pointer(announce_offset, size_of::())? - .cast::(); + .pointer(announce_offset, if m5 { size_of::() } + else { size_of::() })?; let index = |delta: u32| memory::ITEMS[stage] + ((base + delta) % ITEM_RING_ENTRIES) as usize * 8; let items = [ @@ -91,7 +100,9 @@ impl<'a> Lane<'a> { if lead { push(graph.config.gpu_va() + (config_offset + memory::LEAD) as u64); } - push(descriptor); + if !m5 { + push(descriptor); + } if present { push(graph.config.gpu_va() + config_offset as u64); } @@ -105,16 +116,20 @@ impl<'a> Lane<'a> { config, config_destination, config_present: present, - announce: KickAnnounce::new( - config.qid.try_into()?, - if stage == 0 { - DataMaster::Tiling - } else { - DataMaster::Fragment - }, - timestamp, - config.priority as u8, - )?, + announce: if m5 { + Announcement::M5(m5::queue::notification(config.qid.try_into()?, timestamp)?) + } else { + Announcement::Neo(KickAnnounce::new( + config.qid.try_into()?, + if stage == 0 { + DataMaster::Tiling + } else { + DataMaster::Fragment + }, + timestamp, + config.priority as u8, + )?) + }, announce_destination, prefix: count as usize - 1, prefix_end: (base + count - 1) % ITEM_RING_ENTRIES, @@ -146,9 +161,21 @@ impl<'a> Lane<'a> { // SAFETY: The full record is inside the retained queue graph and is // exclusively owned by this ordinal. Publication follows its copy. unsafe { - self.announce_destination.write_unaligned(self.announce); + match &self.announce { + Announcement::Neo(value) => self.announce_destination + .cast::().write_unaligned(*value), + Announcement::M5(value) => self.announce_destination + .cast::().write_unaligned(*value), + } } } + /// Fill the retained records without advancing the firmware-visible cursor. + /// M5's host-port transaction must occur between this and publish_kick. + pub(super) fn prepare(&self) { + self.items(0..self.prefix + 1); + self.config(); + self.announce(); + } pub(super) fn publish_prefix(&self) { self.items(0..self.prefix); self.config(); diff --git a/drivers/gpu/drm/asahi/g17/queue/render/retirement.rs b/drivers/gpu/drm/asahi/g17/queue/render/retirement.rs index 4377bbfe3e311..19a2caabe68e1 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/retirement.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/retirement.rs @@ -39,7 +39,7 @@ pub(crate) struct Window { impl Window { pub(crate) fn new(base: u32, count: u32, capacity: u32) -> Result { - if base >= capacity || !(2..=MAX_GROUP_ITEMS).contains(&count) || count >= capacity { + if base >= capacity || !(1..=MAX_GROUP_ITEMS).contains(&count) || count >= capacity { return Err(EINVAL); } Ok(Self { @@ -230,8 +230,7 @@ pub(crate) fn poll_stampless(ticket: &mut Ticket, memory: &mut impl Memory) -> R if !observation.complete { return Ok(Poll::Pending); } - let [start, end] = memory.pass(ticket.ordinal)?; - Ok(Poll::Complete([start, start, start, end])) + Ok(Poll::Complete(memory.stage_timestamps(ticket.ordinal)?)) } /// Outstanding publications span less than half the 24-bit stamp sequence. @@ -255,6 +254,11 @@ pub(crate) trait Memory { fn snapshot(&mut self, ordinal: u64) -> Result; fn stamps(&mut self) -> Result<[u32; 2]>; fn pass(&mut self, ordinal: u64) -> Result<[u64; 2]>; + /// Four stage timestamps: M5 records them, G17P derives them from its pass. + fn stage_timestamps(&mut self, ordinal: u64) -> Result<[u64; 4]> { + let [start, end] = self.pass(ordinal)?; + Ok([start, start, start, end]) + } } #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -314,8 +318,7 @@ fn complete_poll( // Refresh this generation's pass after observing the post-maintenance // stamps. The pre-stamp snapshot alone cannot authorize export. fence(Ordering::Acquire); - let [start, end] = memory.pass(ticket.ordinal)?; - return Ok(Poll::Complete([start, start, start, end])); + return Ok(Poll::Complete(memory.stage_timestamps(ticket.ordinal)?)); } ticket.previous = snapshot.cursors; Ok(Poll::Pending) diff --git a/drivers/gpu/drm/asahi/g17/recovery.rs b/drivers/gpu/drm/asahi/g17/recovery.rs index d7f8b46eda039..f6decfd7f630e 100644 --- a/drivers/gpu/drm/asahi/g17/recovery.rs +++ b/drivers/gpu/drm/asahi/g17/recovery.rs @@ -11,7 +11,7 @@ mod attribution; mod slot; mod safety; pub(crate) use safety::{note_retired, since_retired}; -pub(crate) use attribution::Sources; +pub(crate) use attribution::{Sources, PAGE_FAULT}; use attribution::Attribution; pub(super) fn slot_key(slot: usize, slot_keys: Option) -> Option { @@ -50,8 +50,8 @@ pub(super) struct Entries { } impl Entries { - /// Returns false at the first invalid record: the table is a packed list and - /// records after it are left over from an earlier restart. + /// Counts a valid record. Returns false for an invalid one, which ends the walk on G17P + /// and is skipped on M5 (see `read_entries`). pub(super) fn include(&mut self, flags: u32) -> bool { if flags & 1 == 0 { return false; @@ -134,6 +134,8 @@ pub(super) trait Memory { fn entry(&self, index: usize) -> Result<(u32, u32)>; /// One millisecond wait; this is polled because the firmware sends no state-3 notification. fn wait_tick(&self); + /// The M5 firmware's recovery table, whose stamp form is not the G17P one. + fn is_m5(&self) -> bool; } #[derive(Copy, Clone, Debug, PartialEq, Eq)] @@ -170,7 +172,11 @@ impl State { pub(super) fn timeout_limit(&mut self, now: u64) -> bool { self.timeouts.note_unless_progress(now, safety::retired()) } - pub(super) fn restart_limit(&mut self, now: u64) -> bool { self.restarts.note(now) } + /// Like timeouts, restarts with work retired between them start a new window: each one + /// still costs the guilty client its device, but only restarts without progress stop the GPU. + pub(super) fn restart_limit(&mut self, now: u64) -> bool { + self.restarts.note_unless_progress(now, safety::retired()) + } /// A restart that only dropped leftover work of failed VMs (`ServiceResult::leftover`) /// does not count toward the storm limit. Returns whether it was forgiven. pub(super) fn forgive_restart(&mut self) -> bool { self.restarts.forgive() } @@ -304,7 +310,14 @@ pub(super) fn service( let (listed, exact, fallback) = (entries.listed, entries.exact, entries.fallback); let (cancelled, blame, leftover) = classify(host, entries, request.blamed, attribution); - if listed != 0 { + if listed != 0 && host.is_m5() { + // The M5 table only classifies (read_entries): each command is retired from its own + // queue's records, replayed when it never ran. + pr_warn!( + "G17: GPU recovery: {} commands were in flight; each is retired from its own queue's records\n", + listed + ); + } else if listed != 0 { pr_warn!( "G17: GPU recovery: {} commands were in flight ({} matched by stamp, {} by queue); they fail instead of completing\n", listed, @@ -331,17 +344,26 @@ pub(super) fn service( Ok(ServiceResult { scan_error, target_error, leftover }) } -/// Walks the packed recovery table up to its first invalid record and marks -/// every listed command interrupted. A failed read keeps what was read so far. +/// Walks the recovery table. G17P's is a packed list: records after its first invalid one are +/// left over from an earlier restart, so the walk ends there. M5's need not be packed, so all +/// of its records are walked and invalid ones skipped; they only classify the recovery, while +/// on G17P every listed command is marked interrupted. A failed read keeps what was read so far. fn read_entries(host: &mut H) -> Entries { let mut entries = Entries::default(); + let m5 = host.is_m5(); for index in 0..RECOVERY_RECORDS { let Ok((flags, stamp)) = host.entry(index) else { break; }; if !entries.include(flags) { + if m5 { + continue; + } break; } + if m5 { + continue; + } let qid = ((flags >> 4) & 0x7f) as u8; match host.note_interrupted(qid, stamp) { Interrupted::Exact => { @@ -459,6 +481,7 @@ mod tests { } impl Memory for Halted { + fn is_m5(&self) -> bool { false } fn recovery_state(&self) -> Result { Ok(self.state.get()) } fn host_recovery(&self) -> Result { Ok(0) } fn fault_sources(&self) -> Result { @@ -513,6 +536,96 @@ mod tests { fn recovery_mut(&mut self) -> &mut State { &mut self.recovery } } + /// A halted recovery table holding `records` (index, flags); every other record is invalid. + /// Records the QIDs whose commands the walk marks interrupted. + struct Table { + recovery: State, + m5: bool, + records: [(usize, u32); 5], + marked: u128, + } + + impl Table { + fn new(m5: bool) -> Self { + const VALID: u32 = 1; + Self { + recovery: State::new(), + m5, + records: [ + (0, VALID | (12 << 4)), + (1, VALID | u32::from(GUILTY) | (17 << 4)), + // Index 2 is invalid: G17P's list ends here, and what follows is left over. + (3, VALID | u32::from(GUILTY | CANCELLED) | (40 << 4)), + (RECOVERY_RECORDS - 1, VALID | u32::from(GUILTY) | (41 << 4)), + (RECOVERY_RECORDS, VALID | u32::from(GUILTY) | (42 << 4)), + ], + marked: 0, + } + } + } + + impl Memory for Table { + fn is_m5(&self) -> bool { self.m5 } + fn recovery_state(&self) -> Result { Ok(3) } + fn host_recovery(&self) -> Result { Ok(0) } + fn fault_sources(&self) -> Result { Err(EIO) } + fn transition(&self, _expected: u32, _next: u32) -> Result { Err(EIO) } + fn clear_timestamps(&self) -> Result { Err(EIO) } + fn entry(&self, index: usize) -> Result<(u32, u32)> { + assert!(index < RECOVERY_RECORDS); + let flags = self.records.iter().find(|(at, _)| *at == index).map_or(0, |(_, flags)| *flags); + Ok((flags, index as u32)) + } + fn wait_tick(&self) {} + } + + impl Queues for Table { + fn render_scheduler_active(&self) -> bool { true } + fn render_pass_started(&self, _qid: u8, _stamp: u64) -> Option { None } + fn publish_recovery_targets(&self) -> Option { None } + fn oldest_spared(&self, _qid: u8) -> Option { None } + fn visit_owners(&self, _qid: u8, _visit: &mut dyn FnMut(&Arc)) {} + fn classify_work(&mut self, _class: &dyn Fn(&Arc) -> Class) -> u128 { 0 } + fn note_interrupted(&mut self, qid: u8, _stamp: u32) -> Interrupted { + self.marked |= 1u128 << qid; + Interrupted::Exact + } + fn settle_blamed_render(&mut self, _qid: u8, _error: Error) {} + fn settle_cancelled_renders(&mut self, _qids: u128) {} + fn rescan_compute(&mut self) -> Result { Ok(()) } + fn release_retired_compute(&mut self) {} + } + + impl Host for Table { + fn recovery(&self) -> &State { &self.recovery } + fn recovery_mut(&mut self) -> &mut State { &mut self.recovery } + } + + #[test] + fn recovery_table_ends_at_its_first_invalid_record() { + let mut host = Table::new(false); + let entries = read_entries(&mut host); + assert_eq!(entries.listed, 2); + assert_eq!(entries.guilty, 1u128 << 17); + assert_eq!(entries.cancelled, 0); + assert_eq!(entries.exact, 2); + assert_eq!(entries.exact_qids, (1u128 << 12) | (1u128 << 17)); + // Leftover records blame and cancel nothing and mark no bystander's command. + assert_eq!(host.marked, (1u128 << 12) | (1u128 << 17)); + } + + #[test] + fn m5_recovery_table_is_walked_past_invalid_records() { + let mut host = Table::new(true); + let entries = read_entries(&mut host); + assert_eq!(entries.listed, 4); + assert_eq!(entries.guilty, (1u128 << 17) | (1u128 << 40) | (1u128 << 41)); + assert_eq!(entries.cancelled, 1u128 << 40); + // The M5 table only classifies. + assert_eq!((entries.exact, entries.fallback, entries.exact_qids), (0, 0, 0)); + assert_eq!(host.marked, 0); + } + #[test] fn pass_evidence_is_frozen_before_the_first_acknowledgement() { let mut host = Halted::new(true); diff --git a/drivers/gpu/drm/asahi/g17/recovery/attribution.rs b/drivers/gpu/drm/asahi/g17/recovery/attribution.rs index fdf0cd62cbd07..9c7f42c839881 100644 --- a/drivers/gpu/drm/asahi/g17/recovery/attribution.rs +++ b/drivers/gpu/drm/asahi/g17/recovery/attribution.rs @@ -4,7 +4,8 @@ pub(super) const SLOTS: usize = 16; const STAMP_MASK: u64 = (1 << 40) - 1; -const PAGE_FAULT: u32 = 3; +/// Firmware recovery reason for a GPU MMU fault. +pub(crate) const PAGE_FAULT: u32 = 3; #[derive(Clone, Copy, Default)] pub(super) struct Source { diff --git a/drivers/gpu/drm/asahi/g17/recovery/safety.rs b/drivers/gpu/drm/asahi/g17/recovery/safety.rs index 5c2ad346c9890..984cb16cdc4da 100644 --- a/drivers/gpu/drm/asahi/g17/recovery/safety.rs +++ b/drivers/gpu/drm/asahi/g17/recovery/safety.rs @@ -115,6 +115,14 @@ mod tests { assert!(state.restart_limit(u64::MAX)); } #[test] + fn restarts_with_work_retired_between_them_do_not_accumulate() { + let mut storm = Storm::new(); + for n in 0..64 { assert!(!storm.note_unless_progress(0, n)); } + for _ in 1..LIMIT { assert!(!storm.note_unless_progress(0, 64)); } + assert!(storm.note_unless_progress(0, 64)); + assert!(storm.note_unless_progress(0, 65)); + } + #[test] fn separated_faults_do_not_accumulate_and_latched_loss_cannot_wrap() { let mut storm = Storm::new(); for n in 0..64 { assert!(!storm.note(n * WINDOW_NS)); } @@ -140,17 +148,6 @@ mod tests { assert!(storm.forgive()); } #[test] - fn recovery_table_ends_at_its_first_invalid_record() { - use crate::g17::recovery::Entries; - let mut entries = Entries::default(); - assert!(entries.include(1 | (12 << 4))); - assert!(entries.include(1 | 2 | (17 << 4))); - assert!(!entries.include(0)); - assert_eq!(entries.listed, 2); - assert_eq!(entries.guilty, 1u128 << 17); - assert_eq!(entries.cancelled, 0); - } - #[test] fn replay_dependency_checks_never_wait_for_event_service() { use crate::g17::dependency::replay_status; assert_eq!(replay_status(0), Err(EBUSY)); diff --git a/drivers/gpu/drm/asahi/g17/render_resources.rs b/drivers/gpu/drm/asahi/g17/render_resources.rs new file mode 100644 index 0000000000000..fb9f6a2bfd504 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/render_resources.rs @@ -0,0 +1,322 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Immutable linked-program storage, shared by commands in one submission. + +use super::fw::m5::{ + pool::PoolGeometry, + pool_budget::{History, RenderBudget, Sizes}, + render_demand::{ComputeStorage, FragmentStorage, RenderDemand, TileStorage, VertexStorage}, +}; +use super::fw::m5::compute::{ScratchRequest, IntersectionTables}; +use crate::util::Reader; +use kernel::{prelude::*, uapi}; + +// SAFETY: The UAPI record contains only fixed-width unsigned integers. +unsafe impl crate::util::AnyBitPattern for uapi::drm_asahi_render_program {} +// SAFETY: The versioned record contains only unsigned integer storage. +unsafe impl crate::util::AnyBitPattern for uapi::drm_asahi_render_program_v2 {} + +pub(crate) struct Programs { + entries: KVec, +} +impl Programs { + pub(crate) fn parse(bytes: &[u8]) -> Result { + let stride = size_of::(); + if bytes.is_empty() || bytes.len() % stride != 0 { return Err(EINVAL); } + let mut entries = KVec::with_capacity(bytes.len() / stride, GFP_KERNEL)?; + let mut reader = Reader::new(bytes); + for _ in 0..bytes.len() / stride { + let entry: uapi::drm_asahi_render_program = reader.read()?; + Self::validate(&entry)?; + entries.push(Self::wrap(entry), GFP_KERNEL)?; + } + Ok(Self { entries }) + } + + pub(crate) fn from_entries(values: &[uapi::drm_asahi_render_program]) -> Result { + if values.is_empty() { return Err(EINVAL); } + let mut entries = KVec::with_capacity(values.len(), GFP_KERNEL)?; + for entry in values { + Self::validate(entry)?; + entries.push(Self::wrap(*entry), GFP_KERNEL)?; + } + Ok(Self { entries }) + } + + fn wrap(legacy: uapi::drm_asahi_render_program) -> uapi::drm_asahi_render_program_v2 { + uapi::drm_asahi_render_program_v2 { legacy, threads: 0, usage_flags: 0, + scs_layout: 0, auxiliary_allocation: 0, reserved: 0, + intersection_pool: 0, intersection_pool_bytes: 0, + intersection_first: 0, intersection_count: 0, + intersection_default_program: 0, intersection_default_control: 0 } + } + + pub(crate) fn parse_v2(bytes: &[u8]) -> Result { + let stride = size_of::(); + if bytes.is_empty() || bytes.len() % stride != 0 { return Err(EINVAL); } + let mut entries = KVec::with_capacity(bytes.len() / stride, GFP_KERNEL)?; + let mut reader = Reader::new(bytes); + for _ in 0..bytes.len() / stride { + let entry: uapi::drm_asahi_render_program_v2 = reader.read()?; + let mesh = entry.usage_flags & (uapi::DRM_ASAHI_RENDER_PROGRAM_MESH + | uapi::DRM_ASAHI_RENDER_PROGRAM_OBJECT) != 0; + let task = entry.usage_flags & crate::file::DRM_ASAHI_RENDER_PROGRAM_TASK != 0; + if entry.reserved != 0 || entry.usage_flags & !0xf != 0 + || entry.usage_flags & 6 == 6 || entry.legacy.flags != 0 + || (task && entry.usage_flags & uapi::DRM_ASAHI_RENDER_PROGRAM_OBJECT == 0) + || entry.legacy.stage > 1 || (mesh && (entry.legacy.stage != 0 + || entry.threads == 0 || entry.legacy.auxiliary_count != 0 + || entry.legacy.imageblock_bytes != 0 || entry.legacy.multisample_bytes != 0)) + || (!mesh && entry.threads != 0) { return Err(EINVAL); } + if !mesh { Self::validate(&entry.legacy)?; } + let scratch = ScratchRequest::new(entry.scs_layout)?; + let intersection = Self::intersection(&entry)?; + if scratch.enabled() != (entry.usage_flags & uapi::DRM_ASAHI_RENDER_PROGRAM_SCS != 0) + || scratch.enabled() != intersection.enabled() + || (!scratch.enabled() && (entry.scs_layout != 0 || entry.auxiliary_allocation != 0)) { + return Err(EINVAL); + } + entries.push(entry, GFP_KERNEL)?; + } + let programs = Self { entries }; + // A render kick has one SCS layout and one intersection-table binding + // per engine, even when several linked programs contribute demand. + programs.stage_scs()?; + Ok(programs) + } + + fn intersection(e: &uapi::drm_asahi_render_program_v2) -> Result { + IntersectionTables::new(e.intersection_pool, e.intersection_pool_bytes, + e.intersection_first, e.intersection_count, + e.intersection_default_program, e.intersection_default_control) + } + + /// Any linked program runs as a hardware mesh or object (task) stage. + pub(crate) fn uses_mesh(&self) -> bool { + self.entries.iter().any(|e| e.usage_flags + & (uapi::DRM_ASAHI_RENDER_PROGRAM_MESH | uapi::DRM_ASAHI_RENDER_PROGRAM_OBJECT) != 0) + } + + /// An API task (object) stage is linked. Only such kicks get the TA mesh + /// rendering buffer: only the object→mesh TA program binds it, never the + /// mesh-single, grid or indirect ones. + pub(crate) fn uses_task(&self) -> bool { + self.entries.iter().any(|e| e.usage_flags & crate::file::DRM_ASAHI_RENDER_PROGRAM_TASK != 0) + } + + /// 0 ordinary, 1 hardware mesh (including the mesh-only passthrough object + /// stage), 2 API task (object) → mesh. + pub(crate) fn mesh_class(&self) -> u8 { + if self.uses_task() { 2 } + else if self.uses_mesh() { 1 } + else { 0 } + } + + pub(crate) fn stage_scs(&self) -> Result<[StageScratch; 2]> { + let mut stages = [StageScratch::NONE; 2]; + for e in &self.entries { + if e.scs_layout == 0 { continue; } + let next = StageScratch { scratch: ScratchRequest::new(e.scs_layout)?, + intersection: Self::intersection(e)?, auxiliary_allocation: e.auxiliary_allocation }; + let out = &mut stages[e.legacy.stage as usize]; + if out.scratch.enabled() && *out != next { return Err(EINVAL); } + *out = next; + } + Ok(stages) + } + + fn validate(entry: &uapi::drm_asahi_render_program) -> Result { + if entry.stage > 1 || entry.flags != 0 + || (entry.stage == 0 && (entry.auxiliary_count != 0 + || entry.threadgroup_bytes != 0 || entry.imageblock_bytes != 0 + || entry.multisample_bytes != 0)) { return Err(EINVAL); } + Ok(()) + } + + fn demands(&self, mut demand: RenderDemand) -> Result<(RenderDemand, [RenderDemand; 2])> { + let mut stages = [demand; 2]; + stages[0].include_vertex(VertexStorage::default()).map_err(|_| EINVAL)?; + let empty = FragmentStorage::default().parameters().map_err(|_| EINVAL)?; + let no_tile = TileStorage::default().parameters(false).map_err(|_| EINVAL)?; + stages[1].include_fragment(empty, no_tile).map_err(|_| EINVAL)?; + demand.include_vertex(VertexStorage::default()).map_err(|_| EINVAL)?; + demand.include_fragment(empty, no_tile).map_err(|_| EINVAL)?; + for e in &self.entries { + let entry = &e.legacy; + if entry.stage == 0 { + let storage = VertexStorage { + primary_bytes: entry.primary_bytes, + secondary_bytes: entry.secondary_bytes, + linked_bytes: entry.linked_bytes, + minimum_primary_bytes: entry.minimum_primary_bytes, + }; + if e.usage_flags & 6 != 0 { + let storage = ComputeStorage { + primary_bytes: storage.primary_bytes, secondary_bytes: storage.secondary_bytes, + linked_bytes: storage.linked_bytes, minimum_primary_bytes: storage.minimum_primary_bytes, + threadgroup_bytes: entry.threadgroup_bytes, threads: e.threads, + }; + demand.include_mesh(storage, 96, 114688).map_err(|_| EINVAL)?; + stages[0].include_mesh(storage, 96, 114688).map_err(|_| EINVAL)?; + } else { + demand.include_vertex(storage).map_err(|_| EINVAL)?; + stages[0].include_vertex(storage).map_err(|_| EINVAL)?; + } + } else { + let fragment = FragmentStorage { + primary_bytes: entry.primary_bytes, + secondary_bytes: entry.secondary_bytes, + linked_bytes: entry.linked_bytes, + minimum_primary_bytes: entry.minimum_primary_bytes, + auxiliary_count: entry.auxiliary_count, + }.parameters().map_err(|_| EINVAL)?; + let tile = TileStorage { + threadgroup_bytes: entry.threadgroup_bytes, + imageblock_bytes: entry.imageblock_bytes, + multisample_bytes: entry.multisample_bytes, + }.parameters(false).map_err(|_| EINVAL)?; + demand.include_fragment(fragment, tile).map_err(|_| EINVAL)?; + stages[1].include_fragment(fragment, tile).map_err(|_| EINVAL)?; + } + } + Ok((demand, stages)) + } +} + +/// Frozen device topology and shared-queue policy. No per-command allocation. +#[derive(Clone, Copy)] +pub(crate) struct M5Policy { + demand: RenderDemand, + budgets: [RenderBudget; 2], + compute_budgets: [RenderBudget; 2], +} + +#[derive(Clone, Copy)] +pub(crate) struct Plan { + pub(crate) sizes: Sizes, + pub(crate) geometry: PoolGeometry, + pub(crate) resource_flags: u64, + pub(crate) pressure: u32, + pub(crate) stages: Option<[StagePlan; 2]>, +} + +#[derive(Clone, Copy)] +pub(crate) struct StagePlan { + pub(crate) scs: StageScratch, + pub(crate) resource_flags: u64, + pub(crate) ptc_max: u32, +} + +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) struct StageScratch { + pub(crate) scratch: ScratchRequest, + pub(crate) intersection: IntersectionTables, + pub(crate) auxiliary_allocation: u32, +} +impl StageScratch { + pub(crate) const NONE: Self = Self { scratch: ScratchRequest::NONE, + intersection: IntersectionTables::NONE, auxiliary_allocation: 0 }; +} + +impl M5Policy { + pub(crate) fn new(id: &crate::regs::G17Id) -> Result { + // Configured G17S context storage per enabled core. This is separate + // from the final per-kick PTC cap selected during command preparation. + const CONTEXT_FLOOR: u32 = 96; + let nominal = id.num_cores.checked_mul(id.num_clusters).ok_or(EOVERFLOW)?; + Ok(Self { + demand: RenderDemand::new(&id.core_masks, nominal, CONTEXT_FLOOR) + .map_err(|_| EINVAL)?, + compute_budgets: [ + RenderBudget::new(&id.core_masks, CONTEXT_FLOOR, 3).map_err(|_| EINVAL)?, + RenderBudget::new(&id.core_masks, CONTEXT_FLOOR, 2).map_err(|_| EINVAL)?, + ], + budgets: [ + RenderBudget::new(&id.core_masks, CONTEXT_FLOOR, 3).map_err(|_| EINVAL)?, + RenderBudget::new(&id.core_masks, CONTEXT_FLOOR, 4).map_err(|_| EINVAL)?, + ], + }) + } + + pub(crate) fn plan_compute(&self, programs: &ComputePrograms, priority: u8) -> Result { + self.plan_compute_scs(programs, priority, 0, 0) + } + + pub(crate) fn plan_compute_scs(&self, programs: &ComputePrograms, priority: u8, + scs_layout: u64, auxiliary_pages: u32) -> Result { + let scratch = super::fw::m5::compute::ScratchRequest::new(scs_layout)?; + // UMA pool class 1 is realtime; class 2 covers all other + // public priorities. Its resident-kick count also depends on engine. + let (budget, active_kicks) = match priority { + 0 => (&self.compute_budgets[0], 3), + 1..=3 => (&self.compute_budgets[1], 2), + _ => return Err(EINVAL), + }; + let mut demand = self.demand; + for entry in &programs.entries { + demand.include_compute(ComputeStorage { + primary_bytes: entry.primary_bytes, + secondary_bytes: entry.secondary_bytes, + linked_bytes: entry.linked_bytes, + minimum_primary_bytes: entry.minimum_primary_bytes, + threadgroup_bytes: entry.threadgroup_bytes, + threads: entry.threads, + }, 96, 114688).map_err(|_| EINVAL)?; + } + let request = budget.compute_scs_requirements( + demand.requirements().map_err(|_| EINVAL)?, scratch.pages(), auxiliary_pages) + .map_err(|_| EINVAL)?; + let sizes = budget.sizes(request, History::default()).map_err(|_| EINVAL)?; + Ok(Plan { + geometry: sizes.geometry().map_err(|_| EINVAL)?, sizes, + resource_flags: demand.resource_flags().map_err(|_| EINVAL)?, + pressure: (demand.pressure() / active_kicks).clamp(96, 512), + stages: None, + }) + } + + pub(crate) fn plan(&self, programs: &Programs, priority: u8) -> Result { + let (budget, active_kicks) = match priority { 0 => (&self.budgets[0], 3), 1..=3 => (&self.budgets[1], 4), + _ => return Err(EINVAL) }; + let (demand, stage_demands) = programs.demands(self.demand)?; + let scs = programs.stage_scs()?; + let stage = |d: RenderDemand, scs| -> Result { Ok(StagePlan { + scs, + resource_flags: d.resource_flags().map_err(|_| EINVAL)?, + ptc_max: (d.pressure() / active_kicks).clamp(96, 512), + }) }; + let request = budget.render_scs_requirements( + demand.requirements().map_err(|_| EINVAL)?, + scs[0].scratch.pages().max(scs[1].scratch.pages()), + scs[0].auxiliary_allocation.max(scs[1].auxiliary_allocation)).map_err(|_| EINVAL)?; + let sizes = budget.sizes(request, History::default()) + .map_err(|_| EINVAL)?; + Ok(Plan { + geometry: sizes.geometry().map_err(|_| EINVAL)?, sizes, + resource_flags: demand.resource_flags().map_err(|_| EINVAL)?, + pressure: demand.pressure(), + stages: Some([stage(stage_demands[0], scs[0])?, stage(stage_demands[1], scs[1])?]), + }) + } +} + +// SAFETY: The record contains only fixed-width unsigned integers. +unsafe impl crate::util::AnyBitPattern for uapi::drm_asahi_compute_program {} + +pub(crate) struct ComputePrograms { + entries: KVec, +} +impl ComputePrograms { + pub(crate) fn parse(bytes: &[u8]) -> Result { + let stride = size_of::(); + if bytes.is_empty() || bytes.len() % stride != 0 { return Err(EINVAL); } + let mut entries = KVec::with_capacity(bytes.len() / stride, GFP_KERNEL)?; + let mut reader = Reader::new(bytes); + for _ in 0..bytes.len() / stride { + let entry: uapi::drm_asahi_compute_program = reader.read()?; + if entry.flags != 0 || entry.threads == 0 { return Err(EINVAL); } + entries.push(entry, GFP_KERNEL)?; + } + Ok(Self { entries }) + } +} diff --git a/drivers/gpu/drm/asahi/g17/reports.rs b/drivers/gpu/drm/asahi/g17/reports.rs index 543612cd211a6..a3bafd969981d 100644 --- a/drivers/gpu/drm/asahi/g17/reports.rs +++ b/drivers/gpu/drm/asahi/g17/reports.rs @@ -5,7 +5,7 @@ //! Writers and readers hold the device mutex; reads never power the GPU up. use super::{ - fw::channels::{EVENT_WORDS, STATISTICS_TYPES}, + fw::channels::EVENT_WORDS, Role, }; use kernel::prelude::*; @@ -19,6 +19,9 @@ struct Sample { words: [u32; EVENT_WORDS], } +/// One sample per statistics tag a family's 64-bit mask can name. +const TAGS: usize = u64::BITS as usize; + pub(super) struct Reports { samples: KVec, } @@ -26,25 +29,27 @@ pub(super) struct Reports { impl Reports { pub(super) fn new() -> Result { // Allocate before firmware starts; receive and debugfs paths allocate nothing. - let mut samples = KVec::with_capacity(64, GFP_KERNEL)?; - for _ in 0..64 { + let mut samples = KVec::with_capacity(2 * TAGS, GFP_KERNEL)?; + for _ in 0..2 * TAGS { samples.push(Sample::default(), GFP_KERNEL)?; } Ok(Self { samples }) } + /// `allowed` is the family's statistics tag mask (`InitData::statistics_types`). pub(super) fn record( &mut self, role: Role, words: [u32; EVENT_WORDS], host_ns: u64, + allowed: u64, admitted: bool, ) { let tag = words[0]; - if tag >= u32::BITS || STATISTICS_TYPES & (1 << tag) == 0 { + if tag >= u64::BITS || allowed & (1u64 << tag) == 0 { return; } - let sample = &mut self.samples[role as usize * 32 + tag as usize]; + let sample = &mut self.samples[role as usize * TAGS + tag as usize]; sample.count = sample.count.saturating_add(1); sample.admitted = sample.admitted.saturating_add(u64::from(admitted)); sample.last_admitted = admitted; @@ -66,8 +71,8 @@ impl Reports { writeln!( f, "role={} tag={:#x} count={} host_ns={} admitted={} last_admitted={} words={:08x?}", - index / 32, - index % 32, + index / TAGS, + index % TAGS, sample.count, sample.host_ns, sample.admitted, @@ -131,25 +136,30 @@ impl kernel::debugfs::Writer for PowerView { mod tests { use super::*; + /// The G17 family's statistics tags, as `InitData::statistics_types` gives them. + const G17: u64 = super::super::fw::channels::STATISTICS_TYPES as u64; + /// M5's, which add tag 32. + const M5: u64 = super::super::fw::m5::STATISTICS_TYPES; + #[test] fn retains_all_words_and_separates_role_and_tag() { let mut reports = Reports::new().unwrap(); let mut words = [0; EVENT_WORDS]; words[0] = 16; words[17] = 0xdeadbeef; - reports.record(Role::Primary, words, 100, false); - reports.record(Role::Primary, words, 101, true); - reports.record(Role::Secondary, words, 102, false); + reports.record(Role::Primary, words, 100, G17, false); + reports.record(Role::Primary, words, 101, G17, true); + reports.record(Role::Secondary, words, 102, G17, false); assert_eq!(reports.samples[16].count, 2); assert_eq!(reports.samples[16].host_ns, 101); assert_eq!(reports.samples[16].words, words); assert_eq!(reports.samples[16].admitted, 1); assert!(reports.samples[16].last_admitted); - assert_eq!(reports.samples[48].count, 1); - assert_eq!(reports.samples[48].admitted, 0); - assert!(!reports.samples[48].last_admitted); + assert_eq!(reports.samples[TAGS + 16].count, 1); + assert_eq!(reports.samples[TAGS + 16].admitted, 0); + assert!(!reports.samples[TAGS + 16].last_admitted); words[0] = 17; - reports.record(Role::Primary, words, 103, true); + reports.record(Role::Primary, words, 103, G17, true); assert_eq!(reports.samples[17].count, 1); assert_eq!(reports.samples[16].count, 2); } @@ -157,11 +167,25 @@ mod tests { #[test] fn rejects_invalid_tags_before_indexing() { let mut reports = Reports::new().unwrap(); - for tag in [0, 15, 29, 31, 32, u32::MAX] { + for tag in [0, 15, 29, 31, 32, 63, 64, u32::MAX] { let mut words = [0; EVENT_WORDS]; words[0] = tag; - reports.record(Role::Primary, words, 1, true); + reports.record(Role::Primary, words, 1, G17, true); } assert!(reports.samples.iter().all(|s| s.count == 0)); } + + #[test] + fn follows_the_family_mask() { + let mut reports = Reports::new().unwrap(); + let mut words = [0; EVENT_WORDS]; + words[0] = 32; + reports.record(Role::Secondary, words, 1, G17, true); + assert_eq!(reports.samples[TAGS + 32].count, 0); + reports.record(Role::Secondary, words, 2, M5, true); + assert_eq!(reports.samples[TAGS + 32].count, 1); + words[0] = 64; + reports.record(Role::Secondary, words, 3, u64::MAX, true); + assert_eq!(reports.samples.iter().filter(|s| s.count != 0).count(), 1); + } } diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index 9a96e1adbe654..94ed6fc391aa1 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -174,6 +174,8 @@ pub(super) struct Registry { pub(super) compute_building: bool, compute_scheduler_building: bool, replay_outer_wait: Option, + /// A compute replay waits for its owner's renders to park or idle (compute_replay). + replay_consumer_wait: Option, pub(super) observations: compute::retirement::Batch, /// Completion-ring slot where the next compute scan starts. completion_cursor: usize, @@ -211,7 +213,7 @@ impl Registry { qids.reserve_initial_render() }), buffers <- BufferIds::new(), - metrics: Arc::new(MetricsIds::new(), GFP_KERNEL)?, + metrics: Arc::new(MetricsIds::for_firmware(variant.m5_commands)?, GFP_KERNEL)?, qos_ids: QosIds::new()?, admission: admission::Pool::with_waiters(kick::queue_limit(variant.qid_floor), queues, renders)?, render: render::RenderState::new(clusters, deflake, descriptor_flags)?, @@ -226,6 +228,7 @@ impl Registry { compute_building: false, compute_scheduler_building: false, replay_outer_wait: None, + replay_consumer_wait: None, observations: compute::retirement::Batch::new()?, completion_cursor: 0, completion_seen: { @@ -264,8 +267,12 @@ impl Registry { /// An installed idle queue keeps its original QID and backing. Mapping a /// replacement context must succeed before its old binding is displaced. /// A vacant queue (USC backing given back) is offered for re-arming only - /// when no populated idle queue exists. - pub(super) fn rebind_compute(&mut self, owner: u64, context: &Arc, required: usize, admission: &Admission<'_>) -> Result { + /// when no populated idle queue of the owner's VM exists. A populated pool + /// serves one VM: a queue whose backing served another VM is not bound. + /// An open owner's idle channel serving another VM is shared for fresh + /// backing; while any queue waits for such a release, none is shared, and + /// the requester waits for the vacant queue. + pub(super) fn rebind_compute(&mut self, owner: u64, context: &Arc, required: super::freelist::ComputeCapacity, admission: &Admission<'_>) -> Result { admission.check()?; if let Some(queue) = self.compute(owner) { return Ok(if queue.matches(owner, queue.qid(), context)? { @@ -283,7 +290,35 @@ impl Registry { if self.qids.compute_fits(self.admission.queues() as usize) { return Ok(Rebind::Fresh); } + if self.compute.iter().flatten().filter_map(|entry| entry.queue.as_deref()) + .any(|queue| queue.vacancy_pending()) + { + return Err(ENOSPC); + } let teardown = &self.teardown; + let vm = context.vm(); + // A released channel whose backing served another VM goes first: vacate it, then + // re-arm it fresh for this one. + let foreign = self + .compute + .iter() + .flatten() + .filter_map(|entry| entry.queue.as_deref()) + .filter(|queue| queue.reusable() && !queue.pool_vacant() && !queue.pool_serves(vm)) + .filter(|queue| !queue.context().is_some_and(|bound| teardown.releasing(bound))) + .map(|queue| (queue.released_at(), queue.qid())) + .min_by_key(|(key, _)| *key) + .map(|(_, qid)| qid); + if let Some(qid) = foreign { + self.compute + .iter_mut() + .flatten() + .filter_map(|entry| entry.queue.as_deref_mut()) + .find(|queue| queue.qid() == qid) + .ok_or(EIO)? + .require_fresh_backing(); + return Err(ENOSPC); + } let idle = self .compute .iter() @@ -291,29 +326,36 @@ impl Registry { .filter_map(|entry| entry.queue.as_deref()) .filter(|queue| queue.shareable()) .filter(|queue| !queue.context().is_some_and(|bound| teardown.releasing(bound))) - .map(|queue| ((!queue.pool_fits(required), queue.last_published()), queue.qid())) + // A channel serving the requester's VM first, then the best fit, then the longest idle. + .map(|queue| ((!queue.pool_serves(vm), !queue.pool_fits(required), queue.last_published()), queue.qid())) .min_by_key(|(key, _)| *key) .map(|(_, qid)| qid); let Some(qid) = idle else { return Err(ENOSPC); }; admission.check()?; - let shared = self - .compute - .iter_mut() - .flatten() - .filter_map(|entry| entry.queue.as_deref_mut()) - .find(|queue| queue.qid() == qid) - .ok_or(EIO)? - .share_idle()?; - if !shared { - return Err(ENOSPC); + { + let mut queue = self + .compute + .iter_mut() + .flatten() + .filter_map(|entry| entry.queue.as_deref_mut()) + .find(|queue| queue.qid() == qid) + .ok_or(EIO)?; + if !queue.share_idle()? { + return Err(ENOSPC); + } + if !queue.pool_serves(vm) { + // Its backing holds another VM's scratch: vacate it, then re-arm fresh. + queue.require_fresh_backing(); + return Err(ENOSPC); + } } Ok(self.bind_reusable(owner, context, required, admission)?.unwrap_or(Rebind::Fresh)) } /// Whether `owner` can bind a channel now, without changing any binding. Owner 0 asks /// only whether some channel is free for a new owner. - fn compute_readiness(&self, owner: u64, context: Option<&Arc>, required: usize) -> Readiness { - if let (Some(context), Some(queue)) = (context, self.compute(owner).filter(|_| owner != 0)) { + fn compute_readiness(&self, owner: u64, query: Option<(&Arc, super::freelist::ComputeCapacity)>) -> Readiness { + if let (Some((context, required)), Some(queue)) = (query, self.compute(owner).filter(|_| owner != 0)) { return if queue.spared_pending() { Readiness::Progress } else if !matches!(queue.matches(owner, queue.qid(), context), Ok(true)) { @@ -344,7 +386,7 @@ impl Registry { /// Binds `owner` to a reusable channel whose last owner released it, or /// names a vacant one to re-arm. None if no channel is reusable. - fn bind_reusable(&mut self, owner: u64, context: &Arc, required: usize, admission: &Admission<'_>) -> Result> { + fn bind_reusable(&mut self, owner: u64, context: &Arc, required: super::freelist::ComputeCapacity, admission: &Admission<'_>) -> Result> { let mut vacant = None; let teardown = &self.teardown; for mut queue in self @@ -362,14 +404,24 @@ impl Registry { continue; } if queue.pool_vacant() { - vacant.get_or_insert(queue.qid()); + if queue.rearm_open(owner) { + vacant.get_or_insert(queue.qid()); + } continue; } - if !queue.pool_fits(required) { continue; } + if !queue.pool_fits(required) || !queue.pool_serves(context.vm()) { continue; } queue.bind_owner(owner, context.clone())?; return Ok(Some(Rebind::Done)); } - Ok(vacant.map(Rebind::Rearm)) + let Some(qid) = vacant else { return Ok(None) }; + // Reserved for this owner until it binds the queue or gives up: other owners wait for + // a channel instead of each building a pool for the same queue. + self.compute.iter_mut().flatten() + .filter_map(|entry| entry.queue.as_deref_mut()) + .find(|queue| queue.qid() == qid) + .ok_or(EIO)? + .reserve_rearm(owner); + Ok(Some(Rebind::Rearm(qid))) } /// Installs fresh USC backing into the vacant queue `qid`; the caller writes @@ -379,7 +431,7 @@ impl Registry { qid: u8, pages: crate::g17::freelist::Pages, ring: compute::DescriptorRing, - ) -> Result<(u16, u64)> { + ) -> Result { let mut queue = self .compute .iter_mut() @@ -390,6 +442,16 @@ impl Registry { queue.rearm(pages, ring) } + /// Ends `owner`'s re-arm reservation of `qid` when it did not bind the queue. + pub(super) fn release_rearm(&mut self, qid: u8, owner: u64) { + if let Some(mut queue) = self.compute.iter_mut().flatten() + .filter_map(|entry| entry.queue.as_deref_mut()) + .find(|queue| queue.qid() == qid) + { + queue.release_rearm(owner); + } + } + pub(super) fn bind_rearmed(&mut self, qid: u8, owner: u64, context: &Arc) -> Result { let teardown = &self.teardown; let mut queue = self @@ -618,6 +680,8 @@ pub(super) enum Readiness { Channel, /// Retirement-bound capacity or reopened publication. Progress, + /// M5: no unclaimed room on the shared work ring. + Outer, } /// Outcome of offering an owner a retained compute queue. @@ -677,11 +741,21 @@ pub(super) struct Backend { /// channel from being shared away between prepare_job and run_job. #[pin] claimed: kernel::sync::Mutex>>, + /// M5: this owner's claims on the work ring M5 TA, fragment and compute share, one per + /// engine, taken while preparing a publication and consumed by it + /// ([`Backend::claim_outer`], [`Backend::publish_claimed`]). + #[pin] + outer_compute: kernel::sync::Mutex>, + #[pin] + outer_render: kernel::sync::Mutex>, /// Render pair admission, shared by every logical queue of the session. admission: Arc, /// The render pair this owner last published to without the device mutex /// (zero for none): a hint, revalidated by every lend. direct_slot: core::sync::atomic::AtomicU32, + /// Render pairs this owner may hold, fixed at creation with its scheduler's + /// credits and its render pool's sizing. + pairs: u8, } impl Backend { @@ -719,6 +793,8 @@ impl Backend { { return Err(super::capacity::refused(super::capacity::Site::LogicalQueues)); } + // The new context may reuse a pooled scheduler page: clear released QoS rows first. + firmware.queues.qos_ids.forget_released(|ids| firmware.port.qos_forget(ids)); let attempt = if firmware.uat.execution_contexts_used() < crate::mmu::MAX_EXECUTION_CONTEXTS { // Creators hold the device mutex, so neither check above can // have changed; the slot only fails on a broken invariant. @@ -798,8 +874,11 @@ impl Backend { closed: core::sync::atomic::AtomicBool::new(false), direct <- kernel::new_mutex!(None, "g17::Backend::direct"), claimed <- kernel::new_mutex!(None, "g17::Backend::claimed"), + outer_compute <- kernel::new_mutex!(None, "g17::Backend::outer_compute"), + outer_render <- kernel::new_mutex!(None, "g17::Backend::outer_render"), admission, direct_slot: core::sync::atomic::AtomicU32::new(0), + pairs: super::pairs::per_owner(), }), GFP_KERNEL, )?, @@ -810,6 +889,11 @@ impl Backend { /// The shared aperture has exactly one builder. Other queues sleep without /// holding the device mutex; recovery joins its lease before closing roots. fn shared_compute(&self, admission: &Admission<'_>) -> Result { + // The shared aperture is G17P compute state. M5 compute queues need only + // their own pools, and the objects it would install are not M5's. + if self.context.is_m5() { + return Ok(()); + } loop { admission.check()?; let lease = self.shared.preparations.enter_until(admission)?; @@ -952,7 +1036,7 @@ impl Backend { Ok(()) } - fn grow_compute(&self, required: usize, admission: &Admission<'_>) -> Result { + fn grow_compute(&self, required: super::freelist::ComputeCapacity, admission: &Admission<'_>) -> Result { admission.check()?; { let state = self.shared.state.lock(); @@ -965,8 +1049,9 @@ impl Backend { let dev = self.shared.drm()?; let uat = self.shared.state.lock().as_ref().ok_or(ENODEV)?.uat.clone(); let alloc = super::object::Allocator { dev: &dev, uat: &uat }; - let mut pages = Some(super::freelist::Pages::compute_sized(&alloc, required)?); - let mut alias = Some(pages.as_ref().ok_or(EIO)?.map_client(self.context.vm())?); + let mut pages = Some(super::freelist::Pages::for_compute(&alloc, required, + self.context.vm().memory_client())?); + let mut alias = Some(pages.as_mut().ok_or(EIO)?.map_client(self.context.vm())?); let mut deferred = DeferredBatch::publication()?; let mut growth = None; let result = (|| loop { @@ -992,7 +1077,7 @@ impl Backend { firmware.primary.notify(crate::g17::MSG_CONTROL_NOTIFY)?; } } - queue.finish_growth(&firmware.init, &mut pages, &mut alias, required).map(Some) + queue.finish_growth(&firmware.init, &mut pages, &mut alias).map(Some) })(); deferred.finish(); match result { @@ -1033,7 +1118,53 @@ impl Backend { result } - fn build_compute(&self, required: usize, admission: &Admission<'_>) -> Result { + /// Re-arms the vacant queue `qid` reserved for this owner with fresh zeroed backing, built + /// without the device mutex, and binds it. Some(true): bound; Some(false): retry (the + /// preparation epoch changed or the queue is no longer re-armable); None: no memory for + /// the backing now. + fn rearm_vacant( + &self, + qid: u8, + required: super::freelist::ComputeCapacity, + alloc: &super::object::Allocator<'_>, + objects: &Arc, + lease: &super::preparation::Lease, + admission: &Admission<'_>, + ) -> Result> { + let built = (|| -> Result<_> { + let pages = crate::g17::freelist::Pages::for_compute(alloc, required, + self.context.vm().memory_client())?; + Ok((pages, compute::DescriptorRing::new(alloc, objects)?)) + })(); + let (pages, ring) = match built { + Err(error) if error == ENOSPC => return Ok(None), + result => result?, + }; + let mut state = self.shared.state.lock(); + admission.check()?; + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + if !lease.is_current() { + return Ok(Some(false)); + } + if let Err(error) = self.admission_open() { drop(state); return Err(error); } + // Leave rejected backing local so it is dropped after the device guard, not in + // rearm(). + let ready = firmware.queues.compute.iter().flatten() + .filter_map(|entry| entry.queue.as_deref()) + .any(|queue| queue.qid() == qid && queue.can_rearm() && queue.rearm_open(self.owner)); + if !ready { drop(state); return Ok(Some(false)); } + let descriptor = match firmware.queues.rearm_compute(qid, pages, ring) { + Err(error) if error == EAGAIN => return Ok(Some(false)), + result => result?, + }; + // The firmware consumed this slot's release before the backing + // was dropped; the row is rewritten before the queue is kicked. + firmware.init.install_compute_pool(descriptor)?; + firmware.queues.bind_rearmed(qid, self.owner, &self.context)?; + Ok(Some(true)) + } + + fn build_compute(&self, required: super::freelist::ComputeCapacity, admission: &Admission<'_>) -> Result { admission.check()?; self.admission_open()?; loop { @@ -1110,47 +1241,28 @@ impl Backend { let reservation = match reservation { Ok(reservation) => reservation, Err(qid) => { - // Fresh zeroed backing is built without the device mutex; the - // queue may have been taken or re-armed by another owner since. - let built = (|| -> Result<_> { - let pages = crate::g17::freelist::Pages::compute_sized(&alloc, required)?; - Ok((pages, compute::DescriptorRing::new(&alloc, &objects)?)) - })(); - let (pages, ring) = match built { - Err(error) if error == ENOSPC => { + // bind_reusable reserved `qid` for this owner: any outcome but binding it + // ends the reservation, so another owner may re-arm it. + let result = self.rearm_vacant(qid, required, &alloc, &objects, &lease, admission); + if !matches!(result, Ok(Some(true))) { + if let Some(firmware) = self.shared.state.lock().as_deref_mut() { + firmware.queues.release_rearm(qid, self.owner); + } + } + match result? { + Some(true) => return Ok(()), + Some(false) => continue, + None => { drop(lease); self.wait_channels(observed, admission, true)?; continue; } - result => result?, - }; - let mut state = self.shared.state.lock(); - admission.check()?; - let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; - if !lease.is_current() { - continue; } - if let Err(error) = self.admission_open() { drop(state); return Err(error); } - // A competing ready owner may have taken/rearmed it while - // backing was built off-lock. Leave rejected backing local - // so it is dropped after the device guard, not in rearm(). - let ready = firmware.queues.compute.iter().flatten() - .filter_map(|entry| entry.queue.as_deref()) - .any(|queue| queue.qid() == qid && queue.can_rearm()); - if !ready { drop(state); continue; } - let (slot, page_list) = match firmware.queues.rearm_compute(qid, pages, ring) { - Err(error) if error == EAGAIN => continue, - result => result?, - }; - // The firmware consumed this slot's release before the backing - // was dropped; the row is rewritten before the queue is kicked. - firmware.init.initialize_compute_pool_blocks(slot, page_list, required)?; - firmware.queues.bind_rearmed(qid, self.owner, &self.context)?; - return Ok(()); } }; let built = (|| { - let pool = compute::PoolBacking::new(&alloc, reservation.id, required)?; + let pool = compute::PoolBacking::new(&alloc, reservation.id, required, + self.context.vm().memory_client())?; { let mut state = self.shared.state.lock(); admission.check()?; @@ -1160,13 +1272,15 @@ impl Backend { } self.admission_open()?; firmware.queues.reserved(reservation)?; - let (slot, page_list) = pool.pool_descriptor()?; - firmware.init.initialize_compute_pool_blocks(slot, page_list, required)?; + firmware.init.install_compute_pool(pool.pool_descriptor()?)?; } let backing = compute::Backing::new(&alloc, pool)?; // The global scheduler is created after pool/kick backing and // retained even if this queue's later descriptor graph fails. - self.compute_scheduler(&alloc, &lease, admission)?; + // It is G17P compute state; M5 has none. + if !self.context.is_m5() { + self.compute_scheduler(&alloc, &lease, admission)?; + } let queue = compute::Queue::new( &alloc, &objects, @@ -1175,7 +1289,9 @@ impl Backend { self.context.clone(), self.shared.variant.pointers, self.shared.operand_state, + self.shared.m5_board, self.shared.variant.compute, + alloc.uat.is_m5(), )?; Ok(KBox::new(queue, GFP_KERNEL)?) })(); @@ -1320,8 +1436,9 @@ impl DeferredBatch { Deferred::Retired(completion, Err(error)) | Deferred::Drained(completion, Err(error)) | Deferred::FailedOwned(completion, error) | Deferred::Unpublished(completion, error) => { completion.note_input_error(*error); }, - Deferred::Retired(completion, Ok(_)) | Deferred::Drained(completion, Ok(_)) => - completion.note_source_retired(), + Deferred::Retired(completion, Ok(_)) + | Deferred::RetiredCompute(completion, _, _) + | Deferred::Drained(completion, Ok(_)) => completion.note_source_retired(), _ => {}, } self.values.push_within_capacity(value).map_err(|_| EIO)?; @@ -1330,6 +1447,7 @@ impl DeferredBatch { // succeeds; failure fences and rejected events are not retirement. if let Some( Deferred::Retired(completion, _) + | Deferred::RetiredCompute(completion, _, _) | Deferred::Drained(completion, _) | Deferred::Release(completion), ) = self.values.last() @@ -1351,6 +1469,7 @@ impl DeferredBatch { let mut changed = false; for value in &self.values { if let Deferred::Retired(completion, Ok(_)) + | Deferred::RetiredCompute(completion, _, _) | Deferred::Drained(completion, Ok(_)) = value { if let Err(error) = completion.check_source_inputs() { @@ -1381,7 +1500,9 @@ impl DeferredBatch { } struct ComputeHost<'a> { + shared: &'a Arc, init: &'a super::initdata::InitData, + regs: &'a crate::regs::G17Resources, port: &'a super::port::Port, outer: Option>, effort: &'a mut super::power::Effort, @@ -1399,6 +1520,10 @@ impl compute::Host for ComputeHost<'_> { )) } fn prepare_compute_shared(&mut self) -> Result { + // The global compute state is G17P's; M5 has none to activate. + if self.init.is_m5() { + return Ok(()); + } self.init.activate_compute() } fn qos_publish( @@ -1418,16 +1543,32 @@ impl compute::Host for ComputeHost<'_> { fn register_queue(&mut self, queue: &super::fw::kick::KsmQueue) -> Result { self.port.register_queues(core::slice::from_ref(queue)) } - fn publish_outer(&mut self, _priority: u8, slot: &super::fw::channels::WorkSlot) -> Result { + fn publish_outer(&mut self, priority: u8, slot: &super::fw::channels::WorkSlot) -> Result { + if self.init.is_m5() { + return self.init.publish_work(priority, slot).map(|_| ()); + } self.port.publish_compute(self.outer.take().ok_or(EIO)?, slot).map(|_| ()) } fn notify(&mut self, message: u64) -> Result { + // Every M5 pipe-2 doorbell (0x83) is noted in the firmware's submission history first, + // as the render doorbell notes its pair. + if self.init.is_m5() && message >> super::MSG_TYPE_SHIFT == 0x83 && message & 3 == 2 { + if let Err(error) = self.init.note_m5_submission_history() { + pr_warn!("asahi: M5 submission history skipped: {:?}\n", error); + } + } self.port.notify(message) } fn note_submission(&mut self) -> Result { + self.shared.note_submitter(); + self.shared.note_gpu_activity(); self.effort.submit(); + self.effort.note_waiters(super::now_ns(), super::fence::waiters()); super::Firmware::apply_shared_effort(self.init, self.effort, self.port, super::power::FULL_EFFORT) } + fn submit_m5(&mut self, submission: &super::fw::m5::queue::HardwareSubmission) -> Result { + self.regs.submit_m5_work(submission) + } } /// Publication into a bound, installed queue through the port alone. The caller checked the @@ -1474,6 +1615,10 @@ impl compute::Host for DirectHost<'_> { fn note_submission(&mut self) -> Result { Ok(()) } + fn submit_m5(&mut self, _submission: &super::fw::m5::queue::HardwareSubmission) -> Result { + // M5 never publishes without the device mutex (see compute_direct). + Err(EINVAL) + } } impl super::Firmware { @@ -1502,6 +1647,10 @@ impl super::Firmware { let mut holds = KVec::with_capacity(sources.sources.len(), GFP_KERNEL)?; let mut native: KVec = KVec::with_capacity(sources.sources.len(), GFP_KERNEL)?; + // M5 shared-ring queues have no completion-wait records: the consumer's kick entry names + // the source's kick, and the firmware orders it. + let m5 = self.init.is_m5(); + let mut waited: u128 = 0; for source in &sources.sources { if source.submission.source_status()? { continue; } let found = self.queues.compute.iter_mut().flatten() @@ -1514,6 +1663,12 @@ impl super::Firmware { let Some((sequence, dependency, stamp_va, event_slot, hold)) = found else { return Err(EAGAIN); }; + waited |= 1u128 << dependency.qid(); + if m5 { + merged.insert(sequence, dependency)?; + holds.push(hold, GFP_KERNEL)?; + continue; + } // The completion-wait record orders the consumer behind its // source; naming the source in the kick entry as well only adds // a second, slower wait and a predecessor record. @@ -1531,7 +1686,7 @@ impl super::Firmware { if packet.is_published() { for queue in self.queues.compute.iter().flatten() .filter_map(|entry| entry.queue.as_deref()) - .filter(|queue| native.iter().any(|wait| wait.dependency.qid() == queue.qid())) + .filter(|queue| waited & (1u128 << queue.qid()) != 0) { for source in &sources.sources { queue.note_input_consumer(source); } } @@ -1554,8 +1709,19 @@ impl super::Firmware { return Err(EAGAIN); } // No descriptor, accounting or inner producer changes precede this - // reservation; dropping it unused returns the work-ring slot. - let outer = self.port.reserve_compute(packet.context.policy().priority())?; + // reservation; dropping it unused returns the work-ring slot. M5 compute takes its + // slot on the work ring M5 render shares, under this mutex (InitData::publish_work): + // a full ring is refused here, before a queue install or kick changes any state. + let outer = if self.init.is_m5() { + if !self.init.work_ready(packet.context.policy().priority(), + super::fw::queue::DataMaster::Compute)? + { + return Err(EAGAIN); + } + None + } else { + Some(self.port.reserve_compute(packet.context.policy().priority())?) + }; let queues = &mut self.queues; let mut queue = queues .compute @@ -1574,9 +1740,11 @@ impl super::Firmware { return Err(EAGAIN); } let mut host = ComputeHost { + shared: &self.primary.state.shared, init: &self.init, + regs: &self.bridge.regs, port: &self.port, - outer: Some(outer), + outer, effort: &mut self.effort, qids: &mut queues.qids, generation: self.recovery.generation(), @@ -1594,7 +1762,7 @@ impl Registry { /// Everything ensure_compute establishes for `owner`: the shared compute /// objects mapped into its VM, and its queue bound to `context` with a /// pool that fits and no spared witness to release. - fn compute_bound(&self, owner: u64, context: &Arc, required: usize) -> Result { + fn compute_bound(&self, owner: u64, context: &Arc, required: super::freelist::ComputeCapacity) -> Result { let Some(queue) = self.compute(owner) else { return Ok(false); }; @@ -1754,9 +1922,20 @@ impl super::Firmware { } impl Backend { - fn ensure_compute(&self, scs_pages: u16, admission: &Admission<'_>) -> Result { + /// The compute pool capacity `command` needs, in its family's terms: the M5 + /// plan's geometry, or the G17P block count of its SCS request. + pub(super) fn compute_capacity(&self, command: &super::command::Validated) -> Result { + use super::freelist::ComputeCapacity; + let super::command::Validated::Compute { scratch, plan, .. } = command else { return Err(EINVAL); }; + if self.context.is_m5() { + return Ok(ComputeCapacity::M5(plan.map(|plan| plan.geometry))); + } + super::fw::compute_pool::required_blocks(self.shared.compute_blocks, self.shared.active_cores, scratch.pages()) + .map(ComputeCapacity::Blocks) + } + + fn ensure_compute(&self, required: super::freelist::ComputeCapacity, admission: &Admission<'_>) -> Result { admission.check()?; - let required = super::fw::compute_pool::required_blocks(self.shared.compute_blocks, self.shared.active_cores, scs_pages)?; let mut deferred = DeferredBatch::publication()?; loop { admission.check()?; @@ -1794,10 +1973,13 @@ impl Backend { fn compute_direct( &self, packet: &Arc, - required: usize, + required: super::freelist::ComputeCapacity, deferred: &mut DeferredBatch, ) -> Option { - if packet.completion.sources().is_some() + // M5 compute is a shared-ring submission on the ring M5 render shares: it is published only + // under the device mutex, never through the classic compute ring producer. + if self.context.is_m5() + || packet.completion.sources().is_some() || self.context.render_seen() || packet.cancel_requested() || packet.completion.status().get() != 0 @@ -1815,6 +1997,7 @@ impl Backend { if !port.note_submission() { return None; } + self.shared.note_gpu_activity(); match packet.completion.work_state().ok()?.acquire(crate::g17::job::Engine::Compute) { Ok(_) => {} Err(error) if error == EBUSY => return None, @@ -1854,26 +2037,96 @@ impl Backend { { return None; } - let super::command::Validated::Compute { scratch, .. } = &packet.command else { return None; }; - let required = super::fw::compute_pool::required_blocks(self.shared.compute_blocks, self.shared.active_cores, scratch.pages()).ok()?; + let required = self.compute_capacity(&packet.command).ok()?; + let outer = self.outer_domain(); loop { let channels = self.shared.channels.observe(); let progress = self.shared.progress.observe(); - let (domain, observed) = match self.compute_readiness(packet, required) { - Readiness::Ready => return None, - Readiness::Channel => (&self.shared.channels, channels), - Readiness::Progress => (&self.shared.progress, progress), + let claimed = outer.as_ref().map(|domain| domain.observe()); + let (domain, observed) = match (self.compute_readiness(packet, required), &outer, claimed) { + (Readiness::Ready, ..) => return None, + (Readiness::Channel, ..) => (&self.shared.channels, channels), + (Readiness::Progress, ..) => (&self.shared.progress, progress), + (Readiness::Outer, Some(domain), Some(observed)) => (domain, observed), + (Readiness::Outer, ..) => return None, }; - self.shared.queue_events(); match domain.wait(observed) { - Ok(Some(fence)) => return Some(fence), + Ok(Some(fence)) => { + // Registered first: the event worker polls outer consumption only for waiters. + self.shared.queue_events(); + return Some(fence); + } Ok(None) => continue, Err(_) => return None, } } } - fn compute_readiness(&self, packet: &Arc, required: usize) -> Readiness { + /// M5: the domain outer-ring claims wait on. None off M5 or without firmware. + fn outer_domain(&self) -> Option> { + self.shared.state.lock().as_deref().and_then(|firmware| firmware.init.outer_available().ok()) + } + + /// M5: holds this owner's claim on the shared work ring for the next compute or render + /// publication, reserving one under the device mutex unless one with credits left is + /// held. False: no unclaimed room; wait on [`Self::outer_domain`] from a sample taken + /// before this call. Always true off M5. + fn claim_outer(&self, firmware: &super::Firmware, engine: crate::g17::job::Engine) -> Result { + if !firmware.init.is_m5() { + return Ok(true); + } + let (held, count) = match engine { + crate::g17::job::Engine::Compute => ( + &self.outer_compute, + firmware.queues.compute(self.owner).map_or(2, |queue| queue.outer_slots()), + ), + // Tiling and fragment: one slot each. + crate::g17::job::Engine::Render => (&self.outer_render, 2), + }; + let mut held = held.lock(); + if held.as_ref().is_some_and(|claim| claim.remaining() != 0) { + return Ok(true); + } + match firmware.init.reserve_outer(self.context.policy().priority(), count) { + Ok(claim) => { + // Replaces only a spent claim, whose drop notifies nobody. + *held = Some(claim); + Ok(true) + } + Err(error) if error == EBUSY => Ok(false), + Err(error) => Err(error), + } + } + + /// Publishes into this owner's compute channel, lending the held outer claim's credits. + fn publish_claimed( + &self, + firmware: &mut super::Firmware, + packet: &Arc, + dependencies: &[super::fw::kick::KickDependency], + deferred: &mut DeferredBatch, + ) -> Result { + let mut held = self.outer_compute.lock(); + let _scope = (*held) + .as_mut() + .filter(|claim| claim.remaining() != 0) + .map(|claim| firmware.init.outer_scope(claim)) + .transpose()?; + firmware.publish_compute(self.owner, packet.clone(), dependencies, deferred) + } + + /// Returns an engine's unspent outer claim after its publication, cancellation or close. + /// Outside the device mutex: returned credits notify waiters. + pub(super) fn release_outer(&self, engine: crate::g17::job::Engine) { + let held = match engine { + crate::g17::job::Engine::Compute => &self.outer_compute, + crate::g17::job::Engine::Render => &self.outer_render, + }; + let claim = held.lock().take(); + drop(claim); + } + + fn compute_readiness(&self, packet: &Arc, required: super::freelist::ComputeCapacity) -> Readiness { match packet.completion.work_state().and_then(|work| work.acquire(crate::g17::job::Engine::Compute)) { Err(error) if error == EBUSY => return Readiness::Progress, Err(_) => return Readiness::Ready, @@ -1894,7 +2147,15 @@ impl Backend { }); if bound { self.hold_claim(Some(cell)); - return Readiness::Ready; + if !self.context.is_m5() { + return Readiness::Ready; + } + let state = self.shared.state.lock(); + let Some(firmware) = state.as_deref() else { return Readiness::Ready; }; + return match self.claim_outer(firmware, crate::g17::job::Engine::Compute) { + Ok(false) => Readiness::Outer, + _ => Readiness::Ready, + }; } } } @@ -1916,7 +2177,13 @@ impl Backend { { return Readiness::Progress; } - firmware.queues.compute_readiness(self.owner, Some(&self.context), required) + match firmware.queues.compute_readiness(self.owner, Some((&self.context, required))) { + Readiness::Ready => match self.claim_outer(firmware, crate::g17::job::Engine::Compute) { + Ok(false) => Readiness::Outer, + _ => Readiness::Ready, + }, + other => other, + } } /// Replaces the prepared packet's channel claim; None releases it after publication, @@ -1995,10 +2262,7 @@ impl Backend { packet: &Arc, deferred: &mut DeferredBatch, ) -> Option { - let super::command::Validated::Compute { scratch, .. } = &packet.command else { - return Some(Err(EINVAL)); - }; - let required = match super::fw::compute_pool::required_blocks(self.shared.compute_blocks, self.shared.active_cores, scratch.pages()) { + let required = match self.compute_capacity(&packet.command) { Ok(required) => required, Err(error) => return Some(Err(error)), }; @@ -2012,8 +2276,7 @@ impl Backend { packet: &Arc, deferred: &mut DeferredBatch, ) -> Result { - let super::command::Validated::Compute { scratch, .. } = &packet.command else { return Err(EINVAL); }; - let required = super::fw::compute_pool::required_blocks(self.shared.compute_blocks, self.shared.active_cores, scratch.pages())?; + let required = self.compute_capacity(&packet.command)?; if packet.cancel_requested() || self.admission_open().is_err() || packet.completion.status().get() != 0 { return Err(render::FALLBACK); } @@ -2046,8 +2309,11 @@ impl Backend { { return Err(render::FALLBACK); } + if !self.claim_outer(firmware, crate::g17::job::Engine::Compute)? { + return Err(render::FALLBACK); + } let dependencies = firmware.queues.render_prefix(self.owner, packet)?; - match firmware.publish_compute(self.owner, packet.clone(), dependencies.as_slice(), deferred) { + match self.publish_claimed(firmware, packet, dependencies.as_slice(), deferred) { Err(error) if (error == EBUSY || error == EAGAIN) && !packet.is_published() => { Err(render::FALLBACK) } @@ -2089,8 +2355,7 @@ impl Backend { deferred: &mut DeferredBatch, ) -> Result { packet.check_dependencies()?; - let super::command::Validated::Compute { scratch, .. } = &packet.command else { return Err(EINVAL); }; - let required = super::fw::compute_pool::required_blocks(self.shared.compute_blocks, self.shared.active_cores, scratch.pages())?; + let required = self.compute_capacity(&packet.command)?; if let Some(result) = self.compute_direct(packet, required, deferred) { return result; } @@ -2101,7 +2366,7 @@ impl Backend { // publication after recovery both signal `progress`. let mut bound = true; let mut demand = None; - let deadline = now_ns().saturating_add(WORK_RING_WAIT_NS); + let deadline = now_ns().saturating_add(super::preparation::QUEUE_WAIT_NS); let check = || { if packet.cancel_requested() { return Err(ECANCELED); } self.admission_open()?; @@ -2110,9 +2375,11 @@ impl Backend { }; let admission = Admission::new(deadline, &check); let progress = &self.shared.progress; + let outer = self.outer_domain(); loop { admission.check()?; let observed = progress.observe(); + let claimed = outer.as_ref().map(|domain| domain.observe()); if !self.sources_published(packet)? { let recovering = self.shared.state.lock().as_deref().is_some_and(|firmware| { firmware.recovery.pending() @@ -2130,7 +2397,7 @@ impl Backend { // admission and publication without changing this logical queue. if !bound { demand.get_or_insert_with(|| ComputeDemand::new(&self.shared)); - Backend::ensure_compute(self, scratch.pages(), &admission)?; + Backend::ensure_compute(self, required, &admission)?; admission.check()?; } let lease = self.shared.preparations.enter_until(&admission)?; @@ -2178,14 +2445,19 @@ impl Backend { drop(lease); continue; } + let held = self.claim_outer(firmware, crate::g17::job::Engine::Compute)?; + if let (false, Some(domain), Some(claimed)) = (held, &outer, claimed) { + // The shared work ring has no unclaimed room: consumption or returned claims + // free it. + drop(state); + drop(lease); + self.shared.queue_events(); + self.wait_release(domain, claimed, &admission)?; + continue; + } let dependencies = firmware.queues.render_prefix(self.owner, packet)?; admission.check()?; - let result = firmware.publish_compute( - self.owner, - packet.clone(), - dependencies.as_slice(), - deferred, - ); + let result = self.publish_claimed(firmware, packet, dependencies.as_slice(), deferred); if !packet.is_published() && (matches!(result, Err(EAGAIN)) || (matches!(result, Err(EBUSY)) @@ -2301,6 +2573,17 @@ impl super::Shared { state.take() }; if let Some(mut firmware) = firmware { + if !firmware.init.has_engine_retirement_witness() { + // Engines may still write into the session's pages after the processors + // stop, and nothing here proves they have stopped: keep the session. Its + // callbacks run this module's code: the start pinned the module, so this + // only confirms the pin. + if super::retain_session().is_err() { + dev_err!(self.dev.as_ref(), "GPU session kept without a module pin\n"); + } + core::mem::forget(firmware); + return; + } firmware.finish_stopped_queues(); // RTKit destruction still dereferences the mailbox provider. drop(firmware); @@ -2355,7 +2638,7 @@ impl super::Firmware { // states) still ends a scheduler wait that has no timeout of its own. if freed || (shared.channels.has_waiters() - && self.queues.compute_readiness(0, None, 0) == Readiness::Ready) + && self.queues.compute_readiness(0, None) == Readiness::Ready) { shared.channels.notify(); } @@ -2528,7 +2811,7 @@ mod shared_capacity_tests { assert!(matches!(registry.reserve_compute(1000), Err(EAGAIN))); assert!(registry.qids.render_fits(slots.len())); for owner in 1..=u64::from(crate::mmu::MAX_EXECUTION_CONTEXTS) { - let lease = registry.admission.acquire_render(owner, None, u64::MAX, || Ok(())).unwrap(); + let lease = registry.admission.acquire_render(owner, 1, None, u64::MAX, || Ok(())).unwrap(); registry.ensure_render_ids(lease.slot(), owner).unwrap(); leases.push(lease, GFP_KERNEL).unwrap(); } @@ -2551,7 +2834,7 @@ mod shared_capacity_tests { let mut leases = KVec::with_capacity(limit as usize, GFP_KERNEL).unwrap(); for owner in 1..=u64::from(limit) { slots.push(registry.admission.reserve_queue().unwrap(), GFP_KERNEL).unwrap(); - let lease = registry.admission.acquire_render(owner, None, u64::MAX, || Ok(())).unwrap(); + let lease = registry.admission.acquire_render(owner, 1, None, u64::MAX, || Ok(())).unwrap(); registry.ensure_render_ids(lease.slot(), owner).unwrap(); leases.push(lease, GFP_KERNEL).unwrap(); } @@ -2570,7 +2853,7 @@ mod shared_capacity_tests { for client in 0..4u64 { for queue in 0..9u64 { let owner = 1 + client * 9 + queue; - let lease = registry.admission.acquire_render(owner, None, u64::MAX, || Ok(())).unwrap(); + let lease = registry.admission.acquire_render(owner, 1, None, u64::MAX, || Ok(())).unwrap(); registry.ensure_render_ids(lease.slot(), owner).unwrap(); leases.push(lease, GFP_KERNEL).unwrap(); } diff --git a/drivers/gpu/drm/asahi/g17/runtime/backend.rs b/drivers/gpu/drm/asahi/g17/runtime/backend.rs index 10085c3fcb922..705325bf6d153 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/backend.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/backend.rs @@ -8,7 +8,7 @@ use crate::g17::{ job::{self, Engine, Packet}, }; use core::sync::atomic::Ordering; -use kernel::{dma_fence::Fence, drm::sched, prelude::*, sync::Arc}; +use kernel::{dma_fence::{Fence, RawDmaFence}, drm::sched, prelude::*, sync::Arc}; impl Backend { pub(super) fn release_backlog(&self, packet: &Packet) { @@ -94,6 +94,8 @@ impl Backend { // Record intent before scanning queues; every later install/rearm checks it. self.closed.store(true, Ordering::Release); self.hold_claim(None); + self.release_outer(Engine::Compute); + self.release_outer(Engine::Render); let mut state = self.shared.state.lock(); let Some(firmware) = (*state).as_deref_mut() else { return Ok(()); @@ -116,6 +118,21 @@ impl Backend { } impl job::Backend for Backend { + fn creator_cluster_scheduling(&self) -> bool { + self.shared.follows_submitter() + } + fn host_producer_waits(&self) -> bool { + self.shared.m5_board.is_some_and(|board| !board.g17d_forms() && !board.completion_inputs()) + } + fn firmware_sources_need_visible_completion(&self) -> bool { + self.shared.m5_board.is_some_and(|board| board.completion_inputs()) + } + fn fence_queue(&self) -> Arc { + self.shared.fence_queue() + } + fn render_pairs(&self) -> u32 { + u32::from(self.pairs) + } fn input_sources(&self, inputs: &[crate::file::SyncItem]) -> Option> { @@ -169,16 +186,49 @@ impl job::Backend for Backend { Some(self.shared.feed.clone()) } + fn supports_submission(&self) -> bool { + self.shared.submission_enabled + } + + fn supports_engine(&self, engine: crate::g17::job::Engine) -> bool { + !self.shared.firmware_test || engine == crate::g17::job::Engine::Compute + } + + fn prepare_command(&self, command: &mut crate::g17::command::Validated) -> Result { + use crate::g17::command::Validated; + let Some(policy) = self.shared.render_policy else { return Ok(()) }; + let priority = self.context.policy().priority(); + match command { + Validated::Render { resources, plan, .. } => { + let programs = resources.as_deref().ok_or(EINVAL)?; + *plan = Some(policy.plan(programs, priority)?); + // The M5 target layout refuses the passes the M5 path cannot run. Derive it + // here, so that SUBMIT returns the refusal before anything is queued. + crate::g17::queue::render::m5_render_submission::layout( + command, self.shared.clusters, self.shared.variant.programs)?; + } + Validated::Compute { resources, plan, m5_scratch, cdm_auxiliary_allocation, .. } => { + if let Some(programs) = resources { + *plan = Some(policy.plan_compute_scs(programs, priority, m5_scratch.layout(), + *cdm_auxiliary_allocation)?); + } else if m5_scratch.enabled() { + // SCS needs the dispatch storage declarations to size its pool. + return Err(EINVAL); + } + } + } + Ok(()) + } + fn render_scs_blocks(&self, pages: [u16; 2]) -> Result { - crate::g17::fw::render_rt::required_blocks(self.shared.active_cores, pages) + crate::g17::fw::render_rt::required_blocks(self.shared.active_cores, pages, self.pairs) } - fn ensure_compute(&self, scs_pages: u16) -> Result { + fn ensure_compute(&self, command: &crate::g17::command::Validated) -> Result { // A reverse timeline chain must be submitted before the host signal. // Validate here, but do not consume physical capacity behind unsignalled // DRM dependencies. compute_packet admits it when the job can run. - crate::g17::fw::compute_pool::required_blocks(self.shared.compute_blocks, self.shared.active_cores, scs_pages) - .map(|_| ()) + self.compute_capacity(command).map(|_| ()) } fn fail_vm(&self, error: Error) { @@ -253,6 +303,7 @@ impl job::Backend for Backend { if packet.engine() == Engine::Compute { self.hold_claim(None); } + self.release_outer(packet.engine()); self.release_backlog(&packet); self.finish_publication(&packet, result, &mut deferred) } @@ -284,6 +335,7 @@ impl job::Backend for Backend { if packet.engine() == Engine::Compute { self.hold_claim(None); } + self.release_outer(packet.engine()); if packet.request_cancel() { Deferred::Unpublished(packet.completion.clone(), ECANCELED).finish(); return; @@ -295,6 +347,33 @@ impl job::Backend for Backend { if result.is_err() { self.stop_after_error(); } + // drm_sched frees the job as soon as this returns while the job's + // completion callback is still on its scheduler fence, so the fence + // must be signalled by now even when the queue could not settle the + // packet (a busy or failed runtime, a settle racing another queue's + // timeout). A completion its physical queue still owns keeps the VM + // pinned until the queue retires it; that later completion is a no-op. + if !packet.completion.signalled() { + packet.completion.fail_while_owned(ECANCELED); + } + // Another thread may have claimed the completion and not yet signalled. + let fence = packet.completion.scheduler_fence(); + // SAFETY: the retained fence stays alive across this bounded wait, + // which holds no device lock. + unsafe { + kernel::bindings::dma_fence_wait_timeout( + fence.raw(), + false, + kernel::time::msecs_to_jiffies(1000) as _, + ) + }; + } + + /// Not on M5: a job that does not finish within its watchdog interval fails then, however + /// recently other work retired. Its compute kill then follows the first expiry. + fn renews_shared(&self) -> bool { + let state = self.shared.state.lock(); + (*state).as_deref().is_none_or(|firmware| !firmware.init.is_m5()) } fn long_job_beside(&self, _packet: &Arc) -> bool { @@ -324,6 +403,14 @@ impl job::Backend for Backend { if let Some(deferred) = deferred.as_mut() { firmware.scan_renders(deferred)?; } + if let Some((older, own)) = self.behind_own_head(packet, &firmware.init, renewals) { + if renewals == 0 { + dev_info!(self.shared.dev.as_ref(), + "Render watchdog owner={} behind its context's older node {:#x} (own {:#x}); the firmware progresses: renewed\n", + self.owner, older, own); + } + return Ok(true); + } // Borrowed-graph and queued-retirement deferrals share the renewal bound. if timeout::transaction_grace(renewals) && firmware.queues.defer_render_timeout(packet, ETIMEDOUT) diff --git a/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs b/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs index 4f519743aed53..dfa343a9ac275 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs @@ -20,6 +20,14 @@ impl Backend { return Ok(Decision::Own); }; firmware.scan_compute(None, &mut deferred)?; + if let Some((older, own)) = self.behind_own_head(packet, &firmware.init, renewals) { + if renewals == 0 { + dev_info!(self.shared.dev.as_ref(), + "Compute watchdog owner={} behind its context's older node {:#x} (own {:#x}); the firmware progresses: renewed\n", + self.owner, older, own); + } + return Ok(Decision::Defer); + } // Each queue is locked on its own: release this one before visiting the others. let Some(own) = firmware.queues.compute(self.owner) .filter(|queue| queue.can_classify_timeout(packet)) @@ -49,6 +57,23 @@ impl Backend { matches!(result, Ok(Decision::Defer)) } + /// The older node of the packet's context ahead of its own, with its own word, when the + /// firmware's progress word moved within the watchdog's interval, the packet's VM has no + /// failed work and the job's deferrals are within budget + /// ([`crate::g17::timeout::behind_own_head`]). The word is read at every poll. Firmware + /// without the word renews nothing by this rule. + pub(super) fn behind_own_head(&self, packet: &Arc, + init: &crate::g17::initdata::InitData, renewals: u8) -> Option<(u32, u32)> { + let work = packet.completion.work_state().ok()?; + let older = work.older_node(); + let progressing = init.m5_progress().is_ok_and(crate::g17::timeout::note_progress); + let status = packet.completion.status(); + let failed = status.get() != 0 || status.reported_error() != 0; + crate::g17::timeout::behind_own_head(packet.is_published(), older, progressing, + failed, renewals) + .then(|| (older.unwrap_or(0), work.word())) + } + /// The caller owns cancellation of a packet that never claimed publication. /// A missing retained packet may already have a healthy callback waiting off-lock. pub(super) fn timeout_compute(&self, packet: &Arc) -> Result { diff --git a/drivers/gpu/drm/asahi/g17/runtime/compute_exit.rs b/drivers/gpu/drm/asahi/g17/runtime/compute_exit.rs index 9e12842f41c02..4b311f31982e1 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/compute_exit.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/compute_exit.rs @@ -9,7 +9,7 @@ use crate::g17::{ fw::channels::{ContextKill, ControlRecord}, queue::compute::exit, recovery::Memory as _, - timeout::LIVENESS_NS, + timeout::{LIVENESS_NS, RECOVERY_WAIT_MS}, Firmware, MSG_CONTROL_NOTIFY, }; use core::sync::atomic::{fence, AtomicU32, AtomicU64, Ordering}; @@ -143,6 +143,9 @@ impl super::Registry { impl Firmware { pub(in crate::g17) fn service_compute_exit(&mut self, deferred: &mut DeferredBatch) -> Result { + // Released contexts' QoS rows stop naming their scheduler pages at the next event + // turn, and at the latest before a new context is created. + self.queues.qos_ids.forget_released(|ids| self.port.qos_forget(ids)); let recovery_closed = self.init.recovery_state()? == 0; let port = &self.port; for entry in self.queues.compute.iter_mut().flatten() { @@ -201,6 +204,28 @@ impl Firmware { ); } } + // M5: a command the watchdog failed is killed once its quarantine delay has passed, + // also while its owner keeps the queue open (`Queue::open_timeout_kill`). + for index in 0..self.queues.compute.len() { + let due = self.queues.compute[index] + .as_ref() + .and_then(|entry| entry.queue.as_deref()) + .and_then(|queue| queue.open_timeout_kill(now)); + let Some((context, age)) = due else { + continue; + }; + if !self.queues.render.teardown_idle(&context)? { + continue; + } + if let Err(error) = self.publish_compute_exit(&context, age, now, &mut pressed) { + dev_err!( + self.primary.state.shared.dev.as_ref(), + "Scheduler {} exit request failed: {:?}\n", + context.id(), + error + ); + } + } Ok(()) } @@ -242,7 +267,7 @@ impl Firmware { } return Ok(()); } - } else if self.kill_unsafe(context, now) { + } else if self.kill_unsafe(context, age, now) { return Ok(()); } let (owner, target) = { @@ -326,8 +351,11 @@ impl Firmware { /// retired on the firmware's fault record, so no compute queue is left to carry a kill, /// yet those pairs are released only after one: without this the context, its pairs and /// its memory stay until the processors stop. + /// + /// M5 has none: it issues a context kill only from a compute queue's exit, so a + /// render-only M5 context is never killed. fn render_kill_candidate(&self, index: usize, context: &Arc, age: u64) -> bool { - render_kill_eligible( + !self.init.is_m5() && render_kill_eligible( age, self.queues.teardown.render_kill_published(index) || context.kill_published(), self.queues.render.executed_slots(context) != 0, @@ -428,8 +456,11 @@ impl Firmware { /// minutes of a five-lane load with 67 faults), even with nothing freed before the kill's /// acknowledgement; such a kill takes the next moment without foreign compute instead. /// Render work is not waited for. A GPU that retired nothing for [`LIVENESS_NS`] has - /// stopped. + /// stopped. The wait is Neo's; M5 publishes its kills at once. fn foreign_compute_running(&self, context: &Arc, now: u64) -> bool { + if self.init.is_m5() { + return false; + } let stalled = crate::g17::recovery::since_retired(now).is_none_or(|since| since >= LIVENESS_NS); !stalled && self.queues.compute.iter().flatten().filter_map(|entry| entry.queue.as_deref()).any(|queue| { @@ -438,13 +469,15 @@ impl Firmware { } /// A compute command of `context` failed and the firmware holds commands behind it that it - /// never runs. `publish_compute_exit` is only reached once `teardown_idle` found all of - /// the context's render pairs idle; the pairs behind such a failure become idle when + /// never runs, or, on M5, the watchdog failed one whose commands may never retire + /// (`Queue::timed_out`). `publish_compute_exit` is only reached once `teardown_idle` found + /// all of the context's render pairs idle; the pairs behind such a failure become idle when /// their commands have executed (`Pair::firmware_drained`), so by then nothing of the /// context runs any more. fn compute_failed(&self, context: &Arc) -> bool { self.queues.compute.iter().flatten().filter_map(|entry| entry.queue.as_deref()).any(|queue| { - queue.context().is_some_and(|bound| Arc::ptr_eq(bound, context)) && queue.fault_pending() + queue.context().is_some_and(|bound| Arc::ptr_eq(bound, context)) + && (queue.fault_pending() || queue.timed_out()) }) } @@ -456,9 +489,13 @@ impl Firmware { /// kill proceeds. A context whose compute failed ([`Self::compute_failed`]) /// is not left alone, since its held commands never retire: its kill waits /// only for other contexts' compute ([`Self::foreign_compute_running`]). - fn kill_unsafe(&self, context: &Arc, now: u64) -> bool { + /// + /// On M5 the wait ends [`RECOVERY_WAIT_MS`] after the context closed (`age`): a closed + /// context's command that never terminates would otherwise hold its queue, pairs and + /// memory for as long as other clients keep the GPU busy. + fn kill_unsafe(&self, context: &Arc, age: u64, now: u64) -> bool { let stalled = crate::g17::recovery::since_retired(now).is_none_or(|since| since >= LIVENESS_NS); - if stalled { + if stalled || (self.init.is_m5() && age >= u64::from(RECOVERY_WAIT_MS) * 1_000_000) { return false; } self.queues.render.foreign_in_flight(context) @@ -480,7 +517,7 @@ impl Firmware { .iter() .flatten() .filter_map(|entry| entry.queue.as_deref()) - .any(|queue| queue.exit_polling(now)) + .any(|queue| queue.exit_polling(now) || queue.open_timeout_polling(now)) { return true; } diff --git a/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs b/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs index 4f6b11eac8d4a..2b4f43b5685ad 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs @@ -6,6 +6,9 @@ use super::DeferredBatch; use crate::g17::{completion::Deferred, recovery, Firmware}; use kernel::prelude::*; +/// Bound for a compute replay waiting on its owner's render abandonment proof. +const CONSUMER_WAIT_NS: u64 = 300_000_000; + impl Firmware { /// Called only from an outer worker or client submission, never from event /// dispatch or a recovery callback nested within publication. All candidate @@ -18,6 +21,7 @@ impl Firmware { if state != 0 { return Ok(()); } + self.queues.replay_consumer_wait = None; let commands = self .queues .compute @@ -53,7 +57,51 @@ impl Firmware { else { break; }; - let outer_ready = self.port.compute_ready(packet.context.policy().priority())?; + // A render consumer named this command through a firmware dependency. + // Republish only while no render of the owner has executed since this recovery + // (none consumed the dead token), after every live render of the owner is + // parked for replay; the render replay then depends on the new command. A + // parked render this command waits through is republished first, so the new + // command depends on it again. + let generation = self.recovery.generation(); + let parked_prefix = owner.is_some_and(|owner| { + self.queues.render.owner_parked_through(owner, packet.order.wait_through[0]) + }); + let (executed, live) = if packet.completion.has_firmware_consumer() { + owner.map_or((true, false), |owner| self.queues.render.owner_render_state(owner, generation)) + } else { + (false, false) + }; + let must_wait = parked_prefix || (!executed && live); + let now = super::now_ns(); + let waited = if must_wait { + self.queues.compute[index] + .as_mut() + .and_then(|entry| entry.queue.as_deref_mut()) + .ok_or(EIO)? + .consumer_wait(now) + } else { + 0 + }; + if must_wait && waited < CONSUMER_WAIT_NS { + // The event worker polls while this waits. + self.queues.replay_consumer_wait = Some(now); + return Ok(()); + } + let consumer = (executed || must_wait).then_some(ENODATA); + self.queues.compute[index] + .as_mut() + .and_then(|entry| entry.queue.as_deref_mut()) + .ok_or(EIO)? + .clear_consumer_wait(); + // M5 compute takes its slot on the work ring M5 render shares, whose unclaimed + // room the publication checks (InitData::work_ready). + let priority = packet.context.policy().priority(); + let outer_ready = if self.init.is_m5() { + self.init.work_ready(priority, crate::g17::fw::queue::DataMaster::Compute)? + } else { + self.port.compute_ready(priority)? + }; if !outer_ready { let now = super::now_ns(); let since = *self.queues.replay_outer_wait.get_or_insert(now); @@ -66,7 +114,9 @@ impl Firmware { self.queues.replay_outer_wait = None; } - let result = if !outer_ready { + let result = if let Some(error) = consumer { + Err(error) + } else if !outer_ready { // This replay has no live firmware ownership. Persistent // backpressure uses the existing failed-replay retirement. Err(EBUSY) diff --git a/drivers/gpu/drm/asahi/g17/runtime/events.rs b/drivers/gpu/drm/asahi/g17/runtime/events.rs index 4af2704be0b44..5114465145081 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/events.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/events.rs @@ -30,6 +30,9 @@ impl event::Transport for Service<'_> { fn roles(&self) -> &'static [Role] { self.firmware.roles() } + fn reports_after_signal(&self) -> bool { + self.firmware.init.is_m5() + } fn trace(&mut self, role: Role) { let firmware = &mut *self.firmware; firmware @@ -130,6 +133,9 @@ impl recovery::Host for Service<'_> { } impl recovery::Memory for Service<'_> { + fn is_m5(&self) -> bool { + self.firmware.init.is_m5() + } fn recovery_state(&self) -> Result { self.firmware.init.recovery_state() } @@ -137,7 +143,30 @@ impl recovery::Memory for Service<'_> { self.firmware.init.host_recovery() } fn fault_sources(&self) -> Result { - self.firmware.init.fault_sources() + let mut mmu_fault = false; + if self.firmware.init.is_m5() { + let registers = self.firmware.bridge.regs.m5_fault_registers(); + pr_err!("asahi: recovery fault registers: {:x?}\n", registers); + // MMU fault status: valid (bit 0), reason (1..3, 0 = unmapped), read + // (4), level (7..8), requestor (9..16), VM slot (17..22); the address + // word holds the faulting address shifted right by 6. + if let Ok([_, status, address, ..]) = registers { + if status & 1 != 0 { + mmu_fault = true; + let access = if status & 0x10 != 0 { "read" } else { "write" }; + pr_err!("asahi: recovery: GPU MMU fault: {} reason {} at {:#x}, level {}, requestor {:#x}, VM slot {}\n", + access, (status >> 1) & 7, + (address & ((1 << 42) - 1)) << 6, (status >> 7) & 3, (status >> 9) & 0xff, + (status >> 17) & 0x3f); + } + } + } + let mut sources = self.firmware.init.fault_sources()?; + // M5 reads no firmware reason word; a latched MMU fault is reason 3. + if mmu_fault && sources.reason.is_none() { + sources.reason = Some(recovery::PAGE_FAULT); + } + Ok(sources) } fn transition(&self, expected: u32, next: u32) -> Result { @@ -301,6 +330,21 @@ impl Firmware { Ok(()) } + /// Drains every role's trace ring, then their statistics rings. On M5 the event worker calls + /// this after it has signalled the completions of its pass: the reports are not scheduling + /// inputs. + fn drain_reports(&mut self) { + let dev = &self.primary.state.shared.dev; + for &role in self.roles() { + self.rings.drain_trace(dev, &self.init, role); + } + for &role in self.roles() { + self.observe_statistics_power(); + self.rings.drain_statistics(&self.primary.state.shared.dev, &self.init, role); + self.observe_statistics_power(); + } + } + pub(super) fn service_runtime_events( &mut self, deferred: &mut DeferredBatch, @@ -440,6 +484,10 @@ impl crate::g17::Shared { if self.crashed.load(core::sync::atomic::Ordering::Acquire) { return Err(EIO); } + if firmware.init.is_m5() { + // The completions above are signalled; the reports that arrived with them come after. + firmware.drain_reports(); + } if recover { firmware.service_runtime_recovery(deferred)?; if !firmware.recovery.pending() { @@ -518,9 +566,14 @@ impl crate::g17::Shared { self.queue_reclaims(); } } + // The firmware may consume M5 work-ring slots without a completion interrupt: while + // a preparation waits for unclaimed room, every pass polls the consumers. + let outer = firmware.init.poll_outer_progress()?; let backpressured = firmware.render_control_backpressured(); let polling = backpressured + || firmware.init.outer_waiting() || firmware.queues.replay_outer_wait.is_some() + || firmware.queues.replay_consumer_wait.is_some() || firmware.queues.render.replay_polling() || firmware.queues.teardown.polling(now) || firmware.compute_exit_polling(now) @@ -570,6 +623,10 @@ impl crate::g17::Shared { if grow_ready { self.queue_grow(); } + // Outside the device mutex: the notification can run DMA-fence callbacks. + if let Some(domain) = outer { + domain.notify(); + } // Firmware ring room, consumed releases and teardown steps have no // event of their own: recheck render publishers' conditions every pass. if self.renders.has_waiters() { diff --git a/drivers/gpu/drm/asahi/g17/runtime/render.rs b/drivers/gpu/drm/asahi/g17/runtime/render.rs index fc2e01be761ae..4b62d96d7cd3c 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render.rs @@ -524,6 +524,8 @@ impl Registry { struct Host<'a> { firmware: &'a mut crate::g17::Firmware, registration_powered: bool, + /// An M5 pair's installation, submitted with its first outer publication. + m5_registration: Option, } impl Host<'_> { fn release_registration(&mut self) { @@ -542,6 +544,9 @@ impl Host<'_> { } } impl crate::g17::queue::render::Host for Host<'_> { + fn generation(&self) -> u64 { + self.firmware.recovery.generation() + } fn epoch(&self) -> Result<(u64, u32)> { Ok(( self.firmware.recovery.generation(), @@ -571,8 +576,18 @@ impl crate::g17::queue::render::Host for Host<'_> { /// Idle descent is held for the whole first install. A point-in-time power /// check let the firmware gate the cores between the host's configuration /// writes while several clients installed their first pairs at once. - fn install_pair(&mut self, registration: &crate::g17::fw::kick::RenderRegistration) -> Result { + /// M5's host command port installs and kicks the pair in one transaction: + /// its registration is retained until the outer publication, without the + /// core-power hold. + fn install_pair(&mut self, registration: &crate::g17::queue::render::Registration) -> Result { use crate::g17::fw::variant::QueueRegistration; + let registration = match registration { + crate::g17::queue::render::Registration::M5(submission) => { + self.m5_registration = Some(*submission); + return Ok(()); + } + crate::g17::queue::render::Registration::Neo(registration) => registration, + }; match self.firmware.variant.registration { QueueRegistration::Firmware => return Ok(()), QueueRegistration::Mailbox => { @@ -611,12 +626,44 @@ impl crate::g17::queue::render::Host for Host<'_> { commit: impl FnOnce() -> Result, late_tiling: impl FnOnce(), ) -> Result<[u32; 2]> { + if self.firmware.init.is_m5() { + let submission = match self.m5_registration.take() { + Some(submission) => submission, + None => crate::g17::fw::m5::queue::HardwareSubmission::advance_pair( + fragment.qid, + tiling.qid, + )?, + }; + return self.firmware.submit_m5_render_pair( + priority, + &[copy_slot(fragment), copy_slot(tiling)], + &submission, + commit, + late_tiling, + true, + ); + } self.firmware .init .publish_work_pair(priority, fragment, tiling, true, commit, late_tiling) } + fn publish_configuration_pair( + &mut self, + priority: u8, + fragment: &crate::g17::fw::channels::WorkSlot, + tiling: &crate::g17::fw::channels::WorkSlot, + prepare: impl FnOnce(), + ) -> Result { + let pair = self.firmware.init.m5_work_pair(priority, fragment, tiling)?; + pair.prepare(); + prepare(); + pair.publish(); + Ok(()) + } fn activate_render_pool(&mut self, id: u16, generation: u64) -> Result { - self.release_registration(); + if !self.firmware.init.is_m5() { + self.release_registration(); + } let result = self.firmware.queues.render.pools.activate(id, generation); if result.is_err() { self.firmware.primary.state.shared.lose_device_quietly(); @@ -639,16 +686,28 @@ impl crate::g17::queue::render::Host for Host<'_> { } result } - /// Pipe 0 (tiling) in bits 1:0, priority class in bits 3:2. + /// Pipe 0 (tiling) in bits 1:0, priority class in bits 3:2. M5 renders + /// use the common compute work ring, pipe 2. fn notify_render(&mut self, priority: u8) -> Result { self.release_registration(); - crate::g17::trace::doorbell(crate::g17::trace::RENDER, || { - self.firmware.bridge.regs.g17_core_power() - }); - let result = self - .firmware - .primary - .notify((0x83 << crate::g17::MSG_TYPE_SHIFT) | u64::from(priority) << 2); + let m5 = self.firmware.init.is_m5(); + if m5 { + // A paired publication is one tiling and one fragment submission. + for _ in 0..2 { + if let Err(error) = self.firmware.init.note_m5_submission_history() { + pr_warn!("asahi: M5 submission history skipped: {:?}\n", error); + } + } + } else { + crate::g17::trace::doorbell(crate::g17::trace::RENDER, || { + self.firmware.bridge.regs.g17_core_power() + }); + } + let result = self.firmware.primary.notify( + (0x83 << crate::g17::MSG_TYPE_SHIFT) + | u64::from(priority) << 2 + | if m5 { 2 } else { 0 }, + ); if result.is_err() { self.firmware.primary.state.shared.lose_device_quietly(); } @@ -684,6 +743,18 @@ impl crate::g17::queue::render::Host for Host<'_> { } } +/// The M5 pair submission takes both work slots as one array. +fn copy_slot(slot: &crate::g17::fw::channels::WorkSlot) -> crate::g17::fw::channels::WorkSlot { + crate::g17::fw::channels::WorkSlot { + unk_00: slot.unk_00, + queue_va: slot.queue_va, + data_master: slot.data_master, + items_end: slot.items_end, + qid: slot.qid, + first: slot.first, + } +} + impl crate::g17::Firmware { /// The pair is returned even when transport, accounting or collection fails. /// No off-lock operation or callback which can re-enter the device runs here. @@ -703,6 +774,7 @@ impl crate::g17::Firmware { let mut host = Host { firmware: self, registration_powered: false, + m5_registration: None, }; let result = operation(&mut pair, &mut host); host.release_registration(); diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/dependencies.rs b/drivers/gpu/drm/asahi/g17/runtime/render/dependencies.rs index e150a10e2c9bc..b31a37dae1fd1 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/dependencies.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/dependencies.rs @@ -10,6 +10,14 @@ impl Registry { &self, owner: u64, packet: &Arc, + ) -> Result { + self.render_prefix_at(owner, packet, packet.order.wait_through[0]) + } + pub(in crate::g17::runtime) fn render_prefix_at( + &self, + owner: u64, + packet: &Arc, + prefix: Option, ) -> Result { let mut frontier = Frontier::new(); let compute = packet.engine() == crate::g17::job::Engine::Compute; @@ -18,7 +26,7 @@ impl Registry { pair.collect_dependencies( owner, packet.completion.status(), - packet.order.wait_through[0], + prefix, &mut frontier, compute, )?; @@ -26,7 +34,7 @@ impl Registry { dependencies.collect( owner, packet.completion.status(), - packet.order.wait_through[0], + prefix, &mut frontier, compute, )?; diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/direct.rs b/drivers/gpu/drm/asahi/g17/runtime/render/direct.rs index 409b948ee6aef..c0bb025d2f639 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/direct.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/direct.rs @@ -45,6 +45,9 @@ struct DirectHost<'a> { } impl crate::g17::queue::render::Host for DirectHost<'_> { + fn generation(&self) -> u64 { + self.generation + } fn next_publication(&mut self) -> Result { Ok(self.port.next_publication()) } @@ -65,7 +68,7 @@ impl crate::g17::queue::render::Host for DirectHost<'_> { fn qos_cancel(&mut self, publication: crate::g17::qos::Publication) -> Result { self.port.qos_cancel(publication) } - fn install_pair(&mut self, _registration: &crate::g17::fw::kick::RenderRegistration) -> Result { + fn install_pair(&mut self, _registration: &crate::g17::queue::render::Registration) -> Result { Err(EAGAIN) } fn publish_qid(&mut self, _id: crate::g17::kick::Id) -> Result { @@ -86,6 +89,16 @@ impl crate::g17::queue::render::Host for DirectHost<'_> { self.port .write_render_pair(self.render, &self.room, fragment, tiling, commit, late_tiling) } + fn publish_configuration_pair( + &mut self, + _priority: u8, + _fragment: &crate::g17::fw::channels::WorkSlot, + _tiling: &crate::g17::fw::channels::WorkSlot, + _prepare: impl FnOnce(), + ) -> Result { + // Only M5 publishes configuration pairs, never on this path. + Err(EINVAL) + } fn activate_render_pool(&mut self, id: u16, generation: u64) -> Result { // The lease leaves preparation when the pair is returned; nothing can // observe it in between because the pair is lent. @@ -186,6 +199,7 @@ impl crate::g17::Firmware { let lease = PublicationLease { free_list_generation: preparation.generation, buffer_token, + buffer_context: None, }; let pair = match self.queues.render.take_pair(slot, owner, packet) { Ok(pair) => pair, @@ -263,8 +277,10 @@ impl Backend { deferred: &mut DeferredBatch, ) -> Option { // A compute prefix needs the compute queue's state; leave it to the - // device-mutex path. - if packet.is_published() + // device-mutex path. M5 renders publish through the device mutex + // only: their host-port transaction is part of the publication. + if packet.context.is_m5() + || packet.is_published() || packet.cancel_requested() || packet.completion.status().get() != 0 || packet.order.wait_through[1].unwrap_or(0) != 0 @@ -281,6 +297,7 @@ impl Backend { if !port.note_submission() { return None; } + self.shared.note_gpu_activity(); match packet.completion.work_state().ok()?.acquire(crate::g17::job::Engine::Render) { Ok(_) => {} Err(error) if error == EBUSY => return None, @@ -320,6 +337,7 @@ impl Backend { self.owner, slot, footprint, + packet.order.wait_through[0], &mut dependencies.hazards, )?; } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs b/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs index 277534328406e..032a73759733b 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs @@ -251,7 +251,7 @@ impl crate::g17::Shared { manager.prepare_growth( &dev, &vm, - crate::g17::buffer::firmware_growth_target(before), + manager.firmware_target(), None, ) } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs b/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs index 1e3354d90dbfb..ab94764580521 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs @@ -122,7 +122,17 @@ impl Pools { } fn stage(&self, init: &InitData, pool: &RenderPool) -> Result { - let words = descriptor_words(init.page_pool_descriptor_table()?, pool.id())?; + if pool.is_m5() { + let words = descriptor_words::<8>(init.page_pool_descriptor_table()?, pool.id())?; + pool.repopulate()?; + let descriptor = pool.m5_descriptor()?; + for (word, value) in words.into_iter().zip(descriptor) { + word.store(value, Ordering::Relaxed); + } + fence(Ordering::SeqCst); + return Ok(()); + } + let words = descriptor_words::<4>(init.page_pool_descriptor_table()?, pool.id())?; pool.repopulate()?; let descriptor = pool .page_pool_state()? @@ -240,8 +250,13 @@ impl Pools { /// A detached pool may capture firmware counters only after its last pair /// stopped using this registration. Callers check all retained pair owners. pub(super) fn save(&self, init: &InitData, pool: &RenderPool) -> Result { + if pool.is_m5() { + let words = descriptor_words::<8>(init.page_pool_descriptor_table()?, pool.id())?; + fence(Ordering::Acquire); + return pool.store_m5_descriptor(words.map(|word| word.load(Ordering::Relaxed))); + } let state = pool.page_pool_state()?; - let words = descriptor_words(init.page_pool_descriptor_table()?, pool.id())?; + let words = descriptor_words::<4>(init.page_pool_descriptor_table()?, pool.id())?; fence(Ordering::Acquire); let descriptor = PagePoolDescriptor { page_list: words[0].load(Ordering::Relaxed), @@ -256,14 +271,12 @@ impl Pools { } } -fn descriptor_words(table: &KernelObject, id: u16) -> Result<[&AtomicU64; 4]> { - let offset = Pools::slot(id)? * size_of::(); - Ok([ - table.dword(offset)?, - table.dword(offset + 8)?, - table.dword(offset + 16)?, - table.dword(offset + 24)?, - ]) +/// The `N` descriptor words of pool `id`: four on G17P, eight on M5. +fn descriptor_words(table: &KernelObject, id: u16) -> Result<[&AtomicU64; N]> { + let offset = Pools::slot(id)? * N * size_of::(); + // Check the full row before constructing the infallible array of views. + table.pointer(offset, N * size_of::())?; + Ok(core::array::from_fn(|i| table.dword(offset + i * 8).unwrap())) } impl Pools { diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs index 62ca7895a86f8..0f9153c5938a7 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs @@ -6,6 +6,7 @@ use super::{Config, Reservation}; use crate::g17::{ buffer::MetricsLease, + freelist::{PoolPin, RenderPool}, preparation::Lease, job::Packet, object::{Allocator, KernelObject, Pool}, @@ -44,6 +45,28 @@ impl BuildResources { fn metrics(&self) -> Result { MetricsLease::new(&self.metrics_ids, &self.metrics, self.metrics_bases) } + /// The render pool geometry the command needs: M5 commands carry a plan. + fn pool_plan(&self, packet: &Packet) -> Result> { + let crate::g17::command::Validated::Render { plan, .. } = &packet.command else { + return Err(EINVAL); + }; + if self.uat.is_m5() != plan.is_some() { + return Err(EINVAL); + } + Ok(*plan) + } + /// M5 maps the render's own metrics entry into the context's VM. Called during + /// off-lock preparation, after the context's bindings exist. + fn bind_metrics( + &self, + context: &crate::g17::context::Context, + metrics: &mut MetricsLease, + ) -> Result { + if self.uat.is_m5() { + metrics.map_client(context.vm())?; + } + Ok(()) + } } #[expect( @@ -78,6 +101,148 @@ struct BuildContext<'a> { } impl super::super::Backend { + /// Reserves scarce M5 pool identities before the pair build transfers a + /// physical render pair: the owner's fitting pool, or the context's pool + /// for the command's plan, pinned. The pin is held across the build; the + /// pair takes its own. + fn claim_render_pool( + &self, + first: &Arc, + lease: &Lease, + ) -> Result<(Arc, Option)> { + let dev = self.shared.drm()?; + let (resources, existing) = { + let state = self.shared.state.lock(); + let firmware = (*state).as_deref().ok_or(ENODEV)?; + let resources = BuildResources::new(firmware)?; + let geometry = resources.pool_plan(first)?.map(|plan| plan.geometry); + let mut existing = None; + for entry in firmware.queues.render.entries.iter().flatten() { + let Some(pair) = entry.pair.as_ref() else { + continue; + }; + if pair.matches(self.owner, &self.context)? + && pair.pool().fits(geometry) + && !pair.pool().vacant() + { + existing = Some(pair.pool().clone()); + break; + } + } + (resources, existing) + }; + let alloc = Allocator { + dev: &dev, + uat: &resources.uat, + }; + let pool = match existing { + Some(pool) => pool, + None => self.context.render_pool( + &alloc, + &resources.ids, + &resources.global, + resources.pool_plan(first)?, + )?, + }; + let pin = self.pin_render_pool(&pool, &resources, lease)?; + Ok((pool, pin)) + } + + /// Pins `pool`'s identity. When every M5 identity is taken, an identity + /// whose final free-list release firmware consumed is reclaimed and its + /// physical storage kept for `pool`. ENOSPC when none is reclaimable: the + /// caller waits for render capacity. + fn pin_render_pool( + &self, + pool: &Arc, + resources: &BuildResources, + lease: &Lease, + ) -> Result> { + let mut reclaimed_pages = None; + loop { + if !lease.is_current() { + return Err(EAGAIN); + } + match pool.pin_id() { + Ok(pin) => { + if let Some(reclaimed) = reclaimed_pages.take() { + pool.cache_reclaimed(reclaimed); + } + return Ok(pin); + } + Err(error) if error == ENOSPC && resources.uat.is_m5() => { + // Repeat the release observation after a failed claim: a + // concurrent pin may have taken the reclaimed identity. + let reclaimed = { + let mut state = self.shared.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + if !lease.is_current() { + return Err(EAGAIN); + } + if firmware.recovery.pending() + || crate::g17::recovery::Memory::recovery_state(&firmware.init)? != 0 + { + return Err(EAGAIN); + } + firmware.queues.render.pools.observe(&firmware.init)?; + resources + .ids + .reclaim_one(|id| firmware.queues.render.pools.released(u32::from(id), 0)) + }; + // Dropped outside the device lock: its mappings may take + // memory-management locks. + drop(reclaimed_pages.take()); + if reclaimed.is_some() { + reclaimed_pages = reclaimed; + continue; + } + self.shared.queue_events(); + return Err(error); + } + Err(error) => return Err(error), + } + } + } + + /// Arms an M5 pool's backing for the pair build. A pool without storage + /// takes the physical storage of a released identity first, without + /// evicting a metadata-only owner. The caller holds the pool's pin. + fn prepare_pool_backing( + &self, + pool: &Arc, + resources: &BuildResources, + lease: &Lease, + client: &crate::mmu::Vm, + ) -> Result { + if resources.uat.is_m5() && pool.needs_storage() { + let reclaimed = { + let mut state = self.shared.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + if !lease.is_current() { + return Err(EAGAIN); + } + if firmware.recovery.pending() + || crate::g17::recovery::Memory::recovery_state(&firmware.init)? != 0 + { + return Err(EAGAIN); + } + firmware.queues.render.pools.observe(&firmware.init)?; + resources + .ids + .reclaim_backing(|id| firmware.queues.render.pools.released(u32::from(id), 0)) + }; + if let Some(reclaimed) = reclaimed { + pool.cache_reclaimed(reclaimed); + } + } + let dev = self.shared.drm()?; + let alloc = Allocator { + dev: &dev, + uat: &resources.uat, + }; + pool.ensure_backing(&alloc, &resources.global, client) + } + fn build_render_pair( &self, slot: u8, @@ -87,6 +252,12 @@ impl super::super::Backend { ) -> Result { let lease = self.enter_preparation(sleep)?; let dev = self.shared.drm()?; + // M5 pool identities are scarce: reserve one before any pair moves. + let _pool_claim = if self.context.is_m5() { + Some(self.claim_render_pool(first, &lease)?) + } else { + None + }; let (resources, build) = { let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; @@ -130,7 +301,12 @@ impl super::super::Backend { ) } else { let pair = entry.pair.as_ref().ok_or(EBUSY)?; - if pair.matches(self.owner, &self.context)? { + // An M5 pool that no longer fits the command's geometry + // is replaced by a same-owner rebind below. + let plan = resources.pool_plan(first)?; + if pair.matches(self.owner, &self.context)? + && pair.pool().fits(plan.map(|p| p.geometry)) + { if !pair.pool().vacant() { return Ok(()); } @@ -140,9 +316,17 @@ impl super::super::Backend { dev: &dev, uat: &resources.uat, }; + if resources.uat.is_m5() { + return self.prepare_pool_backing( + &pool, + &resources, + &lease, + self.context.vm(), + ); + } return pool.ensure_backing(&alloc, &resources.global, self.context.vm()); } - if !pair.reusable()? { + if !pair.bindable(self.owner, &self.context)? { return Err(EBUSY); } // The release of the pair's scheduler is armed or still being @@ -152,7 +336,14 @@ impl super::super::Backend { if firmware.queues.teardown.releasing(pair.context()) { return Err(EBUSY); } - let foreign = !Arc::ptr_eq(pair.context().status(), self.context.status()); + // On M5 a different logical context, even of the same VM, + // gets its own graph; only a pool rebind of the same + // context keeps the graph and its queue records. + let foreign = if resources.uat.is_m5() { + !Arc::ptr_eq(pair.context(), &self.context) + } else { + !Arc::ptr_eq(pair.context().status(), self.context.status()) + }; if foreign { firmware .queues @@ -247,7 +438,7 @@ impl super::super::Backend { ctx: &BuildContext<'_>, reservation: Reservation, pm_generation: u64, - metrics: MetricsLease, + mut metrics: MetricsLease, ) -> Result { let BuildContext { lease, @@ -257,9 +448,16 @@ impl super::super::Backend { first, } = *ctx; let built = (|| -> Result> { - let free_list = - self.context - .render_pool(alloc, &resources.ids, &resources.global)?; + let free_list = self.context.render_pool( + alloc, + &resources.ids, + &resources.global, + resources.pool_plan(first)?, + )?; + resources.bind_metrics(&self.context, &mut metrics)?; + if resources.uat.is_m5() { + self.prepare_pool_backing(&free_list, resources, lease, self.context.vm())?; + } Pair::new( alloc, &resources.objects, @@ -277,6 +475,7 @@ impl super::super::Backend { resources.config.deflake, resources.config.descriptor_flags, self.shared.variant, + self.shared.m5_board, ) })(); let (mut built, error) = match built { @@ -328,7 +527,7 @@ impl super::super::Backend { mut parked: crate::g17::queue::render::parked::ParkedPair, buffer: (u8, u64), pm_generation: u64, - metrics: MetricsLease, + mut metrics: MetricsLease, ) -> Result { let BuildContext { lease, @@ -338,9 +537,17 @@ impl super::super::Backend { first, } = *ctx; let built = (|| -> Result<_> { - let free_list = - self.context - .render_pool(alloc, &resources.ids, &resources.global)?; + parked.clear_for_vm(self.context.vm().id())?; + let free_list = self.context.render_pool( + alloc, + &resources.ids, + &resources.global, + resources.pool_plan(first)?, + )?; + resources.bind_metrics(&self.context, &mut metrics)?; + if resources.uat.is_m5() { + self.prepare_pool_backing(&free_list, resources, lease, self.context.vm())?; + } let prepared = parked.prepare( alloc, &resources.objects, @@ -444,7 +651,7 @@ impl super::super::Backend { mut pair: KBox, buffer: Option<(u8, u64)>, pm_generation: Option, - metrics: Option, + mut metrics: Option, ) -> Result { let BuildContext { lease, @@ -454,9 +661,18 @@ impl super::super::Backend { first, } = *ctx; let prepared = (|| { - let free_list = - self.context - .render_pool(alloc, &resources.ids, &resources.global)?; + let free_list = self.context.render_pool( + alloc, + &resources.ids, + &resources.global, + resources.pool_plan(first)?, + )?; + if let Some(metrics) = metrics.as_mut() { + resources.bind_metrics(&self.context, metrics)?; + } + if resources.uat.is_m5() { + self.prepare_pool_backing(&free_list, resources, lease, self.context.vm())?; + } pair.prepare_bind( alloc, &resources.objects, @@ -499,11 +715,15 @@ impl super::super::Backend { { return Err(crate::g17::capacity::refused(crate::g17::capacity::Site::RetiredRenderBindings)); } - firmware.queues.teardown.displace( - pair.context(), - true, - super::super::now_ns(), - )?; + // A rebind of the same context (a render that needs a bigger pool) keeps its + // live scheduler state; only another context's is displaced. + if !pair.matches(self.owner, &self.context)? { + firmware.queues.teardown.displace( + pair.context(), + true, + super::super::now_ns(), + )?; + } pair.bind_owner( prepared.as_mut().ok_or(EIO)?, alloc, @@ -593,8 +813,8 @@ impl super::super::Backend { pub(super) fn ensure_render(&self, packet: &Arc, slot: u8, sleep: bool) -> Result { self.build_render_pair(slot, packet, false, sleep)?; - // An owner has exactly one pair (kick.rs asserts it): no sibling to warm. - if crate::hw::t8140::queues::RENDER_PAIRS_PER_OWNER == 1 { + // An owner held to one pair has no sibling to warm. + if self.pairs == 1 { return Ok(()); } let sibling = { @@ -625,7 +845,9 @@ impl super::super::Backend { packet: &Arc, deferred: &mut super::super::DeferredBatch, sleep: bool, + outer_wait: &mut Option<(Arc, u64)>, ) -> Result { + *outer_wait = None; let lease = self.enter_preparation(sleep)?; let dev = self.shared.drm()?; let (uat, mut pair, addresses, growth_target) = { @@ -648,6 +870,15 @@ impl super::super::Backend { firmware.init.stamp_addresses(qids[1])?, ], }; + // M5: the pair's two slots on the shared work ring are claimed before preparing + // and lent to its publication. Sampled first: a miss waits for the next return. + if let Ok(domain) = firmware.init.outer_available() { + let observed = domain.observe(); + if !self.claim_outer(firmware, crate::g17::job::Engine::Render)? { + *outer_wait = Some((domain, observed)); + return Err(EBUSY); + } + } let pair = firmware .queues .render @@ -737,6 +968,12 @@ impl super::super::Backend { if packet.cancel_requested() || firmware.queues.render.entry(slot)?.closed { return Err(ECANCELED); } + let mut held = self.outer_render.lock(); + let _scope = (*held) + .as_mut() + .filter(|claim| claim.remaining() != 0) + .map(|claim| firmware.init.outer_scope(claim)) + .transpose()?; firmware.publish_render(self.owner, slot, packet, deferred) })(); let result = (|| { diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/replay.rs b/drivers/gpu/drm/asahi/g17/runtime/render/replay.rs index c4fb20fa0952a..67488b7299bbc 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/replay.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/replay.rs @@ -10,6 +10,29 @@ use crate::g17::{object::Allocator, queue::render::RenderAddresses, Firmware}; use kernel::prelude::*; impl super::RenderState { + /// (an executed group of the owner retired in `generation` or later, + /// a published group of the owner is still live and not parked). + pub(in crate::g17) fn owner_render_state(&self, owner: u64, generation: u64) -> (bool, bool) { + let mut executed = false; + let mut live = false; + for pair in self.entries.iter().flatten().filter_map(|entry| entry.pair.as_ref()) { + if pair.owner() != Some(owner) { + continue; + } + executed |= pair.executed_since(generation); + live |= pair.live_unparked(); + } + (executed, live) + } + /// A parked render of the owner precedes a command waiting through `through`. + pub(in crate::g17) fn owner_parked_through(&self, owner: u64, through: Option) -> bool { + self.entries + .iter() + .flatten() + .filter_map(|entry| entry.pair.as_ref()) + .any(|pair| pair.owner() == Some(owner) && pair.replay_through(through)) + } + pub(in crate::g17) fn replay_polling(&self) -> bool { self.entries .iter() diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs index c6c6b615ecd1a..431dbbd748ecf 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs @@ -87,7 +87,7 @@ impl Backend { if !sleep { return Err(FALLBACK); } - match pool.acquire_render(self.owner, Request::of(packet), deadline_ns, || { + match pool.acquire_render(self.owner, self.pairs.into(), Request::of(packet), deadline_ns, || { // Atomics only: the callback runs under the pool mutex. if packet.cancel_requested() || self.closed.load(Ordering::Acquire) { return Err(ECANCELED); @@ -262,11 +262,23 @@ impl Backend { observed: &mut Option, sleep: bool, packet: &Packet, + ) -> Result { + self.wait_render_retirement(clock, observed, sleep, packet, super::super::WORK_RING_WAIT_NS) + } + + /// [`Self::wait_render_busy`] bounded at `bound_ns`. + fn wait_render_retirement( + &self, + clock: &mut WaitClock, + observed: &mut Option, + sleep: bool, + packet: &Packet, + bound_ns: u64, ) -> Result { if !sleep { return Err(FALLBACK); } - let deadline = self.wait_deadline(clock, super::super::WORK_RING_WAIT_NS)?; + let deadline = self.wait_deadline(clock, bound_ns)?; // Sample, then retry once: only a refusal after the sample waits. let Some(sample) = observed.take() else { *observed = Some(self.shared.renders.observe()); @@ -392,7 +404,7 @@ impl Backend { Some(lease) => Some(lease), None if known != 0 => self .admission - .try_acquire_render(self.owner, Request::of(packet)) + .try_acquire_render(self.owner, self.pairs.into(), Request::of(packet)) .ok() .flatten(), None => None, @@ -422,6 +434,8 @@ impl Backend { let mut recovery_since = None; let mut seen = None; let mut states = None; + let mut outer_wait = None; + let mut channel = WaitClock::default(); loop { if packet.cancel_requested() { return Err(ECANCELED); @@ -489,7 +503,16 @@ impl Backend { } result => result?, } - match self.prepare_render(slot, packet, deferred, sleep) { + match self.prepare_render(slot, packet, deferred, sleep, &mut outer_wait) { + Err(error) + if !packet.is_published() && error == crate::g17::context::RENDER_CHANNEL_PENDING => + { + // Both of the context's render channels serve other pairs: one moves here + // once its publications retire. Like a compute queue, it frees only as this + // context's earlier work retires, so wait as long. + self.wait_render_retirement(&mut channel, &mut seen, sleep, packet, + crate::g17::preparation::QUEUE_WAIT_NS)?; + } Err(error) if !packet.is_published() && error == ENOSPC => { self.wait_render_capacity(&mut seen, &mut capacity, sleep)?; continue; @@ -516,6 +539,15 @@ impl Backend { self.shared .preparations .wait_closure(gate, since.saturating_add(recovery_wait_ns()))?; + } else if let Some((domain, observed)) = outer_wait.take() { + // No unclaimed room on the shared work ring: consumption or a + // returned claim frees it. + if !sleep { + return Err(FALLBACK); + } + let deadline = self.wait_deadline(&mut busy, super::super::WORK_RING_WAIT_NS)?; + self.shared.queue_events(); + self.wait_domain(&domain, observed, deadline, packet)?; } else if error == EBUSY { self.wait_render_busy(&mut busy, &mut seen, sleep, packet)?; } else if !sleep { @@ -531,7 +563,7 @@ impl Backend { if self.backlog.load(Ordering::Acquire) != 0 || packet.has_dependencies() { return false; } - let admission = match self.admission.try_acquire_render(self.owner, Request::of(packet)) { + let admission = match self.admission.try_acquire_render(self.owner, self.pairs.into(), Request::of(packet)) { Ok(Some(lease)) => lease, _ => return false, }; @@ -562,6 +594,7 @@ impl Backend { packet.abandon_publication(); return false; } + self.release_outer(crate::g17::job::Engine::Render); let result = self .finish_publication(packet, result, &mut deferred) .map(|_| ()); @@ -586,9 +619,22 @@ impl Registry { .ok_or(EOVERFLOW)? = *dependency; dependencies.render_count += 1; } - if dependencies.render_count != 0 - && matches!(&packet.command, crate::g17::command::Validated::Render { pass, .. } - if pass.fragment_barriers) + let staged = matches!(&packet.command, crate::g17::command::Validated::Render { pass, .. } + if pass.stage_prefixes); + if staged { + let tiling = self.render_prefix_at(owner, packet, packet.order.tiling_wait_through[0])?; + for dependency in tiling.as_slice() { + *dependencies + .render_tiling + .get_mut(dependencies.render_tiling_count) + .ok_or(EOVERFLOW)? = *dependency; + dependencies.render_tiling_count += 1; + } + } + if staged + || (dependencies.render_count != 0 + && matches!(&packet.command, crate::g17::command::Validated::Render { pass, .. } + if pass.fragment_barriers)) { let tiling = self.tiling_prefix(owner, packet)?; for dependency in tiling.as_slice() { @@ -603,9 +649,20 @@ impl Registry { dependencies.compute = queue.dependency( owner, packet.completion.status(), - packet.order.wait_through[1], + packet.order.tiling_wait_through[1], state, )?; + if packet.order.tiling_wait_through[1] != packet.order.wait_through[1] { + let whole = queue.dependency( + owner, + packet.completion.status(), + packet.order.wait_through[1], + state, + )?; + if whole != dependencies.compute { + dependencies.fragment_compute = whole; + } + } } Ok(dependencies) } @@ -704,6 +761,7 @@ impl crate::g17::Firmware { owner, slot, footprint, + packet.order.wait_through[0], &mut dependencies.hazards, )?; } @@ -718,7 +776,14 @@ impl crate::g17::Firmware { } let pool = pair.pool().clone(); let buffer_id = pair.buffer_id(); - let (buffer_id, buffer_token) = self.queues.buffers.acquire_owner(buffer_id)?; + // An M5 PB token is bound to the hardware context and PB state it names. + let buffer_context = pair.m5_pb_state_va().map(|pb_va| (packet.context.id(), pb_va)); + let (buffer_id, buffer_token) = match buffer_context { + Some((context, pb_va)) => { + self.queues.buffers.acquire_owner_bound(buffer_id, context, pb_va)? + } + None => self.queues.buffers.acquire_owner(buffer_id)?, + }; let preparation = match self.queues.render.pools.prepare(&self.init, &pool) { Ok(preparation) => preparation, Err(error) => { @@ -729,6 +794,7 @@ impl crate::g17::Firmware { let lease = PublicationLease { free_list_generation: preparation.generation, buffer_token, + buffer_context: buffer_context.map(|(context, _)| context), }; let result = self.with_render(usize::from(slot), |pair, host| { pair.publish( diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/teardown.rs b/drivers/gpu/drm/asahi/g17/runtime/render/teardown.rs index fffa6d28dd289..5b8379078b151 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/teardown.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/teardown.rs @@ -16,7 +16,21 @@ impl RenderState { self.pools.observe(init) } + /// Nothing of `context` still runs on its render pairs, so a kill of it may be published. + /// A pair whose commands all ran unstamped (`Pair::executed_for_kill`) counts as idle: only + /// that kill releases it. pub(in crate::g17::runtime) fn teardown_idle(&self, context: &Arc) -> Result { + self.idle_for(context, false) + } + + /// As [`Self::teardown_idle`], for releasing `context`'s scheduler: a pair whose commands ran + /// unstamped counts only once a kill of the context has been published, since until then + /// the firmware still links that work. + pub(in crate::g17::runtime) fn release_idle(&self, context: &Arc) -> Result { + self.idle_for(context, true) + } + + fn idle_for(&self, context: &Arc, release: bool) -> Result { // A pair that changed owner keeps running this scheduler's queues until // the new owner's first command leaves it, whichever context owns it now. let scheduler = Some(context.scheduler_va()); @@ -38,7 +52,9 @@ impl RenderState { return Ok(false); } if let Some(pair) = entry.pair.as_ref() { - if !pair.idle_for_context(context)? && !pair.executed_for_kill(context) { + if !pair.idle_for_context(context)? + && !((!release || context.kill_published()) && pair.executed_for_kill(context)) + { return Ok(false); } } @@ -77,7 +93,13 @@ impl RenderState { } let mut index = 0; while index < self.retired.len() { - if self.retired[index].buffer_id().is_some() { + // An M5 pool replaced by a same-context rebind stays until + // firmware acknowledged its release. + let pool = self.retired[index].pool(); + if self.retired[index].buffer_id().is_some() + || (self.retired[index].wait_for_pool_release() + && !self.pools.released(u32::from(pool.id()), pool.control_va())) + { index += 1; continue; } diff --git a/drivers/gpu/drm/asahi/g17/runtime/teardown.rs b/drivers/gpu/drm/asahi/g17/runtime/teardown.rs index 88252103b676b..cbaf5c6e64251 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/teardown.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/teardown.rs @@ -7,7 +7,7 @@ use crate::g17::{ buffer::BufferIds, channel::Rings, context::Context, - fw::channels::{ControlRecord, SchedulerStateRelease}, + fw::channels::ControlRecord, initdata::InitData, queue::render::RetiredBinding, recovery::Memory as _, @@ -111,7 +111,7 @@ struct Service<'a> { impl Host for Service<'_> { fn render_idle(&mut self, context: &Arc) -> Result { - self.render.teardown_idle(context) + self.render.release_idle(context) } fn compute_idle(&self, context: &Arc) -> bool { self.compute @@ -126,13 +126,10 @@ impl Host for Service<'_> { fn control_backpressured(&self) -> bool { self.render.control_backpressured() } - fn publish(&mut self, record: &SchedulerStateRelease) -> Result { + fn publish_release(&mut self, page_va: u64, id: [u8; 4]) -> Result { Rings::publish_control( self.init, - &ControlRecord::SchedulerStateRelease(SchedulerStateRelease::new( - record.page_va, - record.id, - )), + &ControlRecord::SchedulerStateRelease(self.init.scheduler_release(page_va, id)), ) } fn notify(&mut self) -> Result { diff --git a/drivers/gpu/drm/asahi/g17/status.rs b/drivers/gpu/drm/asahi/g17/status.rs index db9dffc5b3d4b..c0d905b83de06 100644 --- a/drivers/gpu/drm/asahi/g17/status.rs +++ b/drivers/gpu/drm/asahi/g17/status.rs @@ -78,6 +78,11 @@ impl VmStatus { self.recovery_guilty.load(Ordering::Acquire) } + /// The client's VM handle is gone (`release_mirror`): the client submits nothing more. + pub(crate) fn client_closed(&self) -> bool { + self.client_closed.load(Ordering::Acquire) + } + /// Record permanent admission failure and report it to the client. #[track_caller] pub(crate) fn record(&self, error: Error) { diff --git a/drivers/gpu/drm/asahi/g17/teardown.rs b/drivers/gpu/drm/asahi/g17/teardown.rs index fa2a72a77f060..39208dc7b3f34 100644 --- a/drivers/gpu/drm/asahi/g17/teardown.rs +++ b/drivers/gpu/drm/asahi/g17/teardown.rs @@ -4,7 +4,7 @@ //! private-memory release and control-ring consumption are separate ownership witnesses. //! The device mutex serializes this state with the Registry's displaced graph ownership. -use super::{context::Context, freelist::control_consumed, fw::channels::SchedulerStateRelease}; +use super::{context::Context, freelist::control_consumed}; use core::sync::atomic::{fence, Ordering}; use kernel::{prelude::*, sync::Arc}; @@ -82,7 +82,9 @@ pub(crate) trait Host { fn compute_idle(&self, context: &Arc) -> bool; fn flists_released(&self, context: &Arc) -> bool; fn control_backpressured(&self) -> bool; - fn publish(&mut self, record: &SchedulerStateRelease) -> Result; + /// Publishes the release of the scheduler page at `page_va`, identity bytes in record + /// order, in the family's record. + fn publish_release(&mut self, page_va: u64, id: [u8; 4]) -> Result; fn notify(&mut self) -> Result; fn detach(&mut self, context: &Arc) -> Result; fn report(&mut self, context: &Arc, failure: Failure); @@ -215,11 +217,7 @@ impl Pending { } let result = (|| { let id = entry.context.release_identity()?; - let record = SchedulerStateRelease::new( - entry.context.scheduler_va(), - [id[3], id[0], id[1], id[2]], - ); - host.publish(&record) + host.publish_release(entry.context.scheduler_va(), [id[3], id[0], id[1], id[2]]) })(); let next = match result { Ok(next) => next, diff --git a/drivers/gpu/drm/asahi/g17/thermal.rs b/drivers/gpu/drm/asahi/g17/thermal.rs new file mode 100644 index 0000000000000..6dd9bc7c2b15a --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/thermal.rs @@ -0,0 +1,772 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! GPU power budget governor for M5 (J714). +//! +//! The firmware limits the GPU to the host power budget word in Region C within about a second +//! (the calibrated maximum is written there at initialization). Left at that maximum under +//! sustained load, the SoC hotspot temperature computed by the SMC (about 6 C above the hottest +//! GPU die sensor while the GPU dominates) spends time above roughly 97 C, and the firmware's +//! emergency path then pins the GPU to its base performance state for 20-40 s, repeatedly. This +//! governor reads the hotspot temperature through the thermal framework once per second while the +//! GPU is busy and, once it reaches the engage temperature, lowers the budget with a PI controller +//! (plus a feed-forward on the temperature's rise) so it settles at the target, below that region. +//! +//! The controller works below the GPU's actual draw (the calibrated maximum is far above it). It +//! keeps the integral it converged to across releases, so after a release or a firmware clamp it +//! resumes from that budget; after a long period without control the memory decays to the +//! cold-start budget, never to the ceiling. It holds while the firmware's emergency clamp is +//! active and never releases then. The default target keeps the hotspot below the level at which +//! the firmware clamps even with a heat-soaked chassis. As a safety net, each emergency clamp the +//! governor sees lowers its effective target by 1 C (at most 6 C, not below 84 C), and every 10 +//! minutes without one raises it again by 1 C. It publishes the budget like the effort input, only when it changes, and +//! restores the calibrated maximum on release, when the GPU goes idle, when no sensor can be read +//! and at shutdown. Below the engage temperature nothing is written. + +use core::ffi::{c_char, c_int, c_uint, c_void, CStr}; +use core::sync::atomic::{AtomicI32, Ordering}; +use kernel::{bindings, prelude::*}; + +// fans.c: the SMC hwmon cell's fan requests and thermal zone reads, which have stubs where +// their drivers are not built. +extern "C" { + fn asahi_zone_temp(name: *const c_char, temp: *mut c_int) -> c_int; + fn asahi_fans_request(dev: *mut c_void, permille: c_uint, min_step_rpm: c_uint) -> c_int; +} + +/// SoC hotspot temperature computed by the SMC (registered as a thermal zone by the SMC driver): +/// the input of the firmware's emergency clamp. +const J714_HOTSPOT_ZONE: &CStr = c"smc-TCMb"; +/// The hottest of the 42 SMC GPU die sensors of J714 near the limit, and their offset to the +/// hotspot while the GPU dominates: the fallback input when the hotspot cannot be read. +const J714_GPU_DIE_ZONES: [&CStr; 8] = [ + c"smc-Tg08", c"smc-Tg12", c"smc-Tg0O", c"smc-Tg0X", c"smc-Tg29", c"smc-Tg2P", c"smc-Tg1U", + c"smc-Tg43", +]; +const GPU_DIE_TO_HOTSPOT_MC: i32 = 6000; +/// Where the SMC driver discovers the temperatures and the device tree names the hotspot key +/// (TCMb) as the SoC die zone's only key (J775D), that zone reads the hotspot. +const SOC_DIE_ZONE: &CStr = c"macsmc_soc_die"; + +/// Region C word holding the host GPU power budget (mW). +pub(crate) const REGION_C_POWER_BUDGET: usize = 0x88; +/// Shared power-policy word holding the GPU's current power draw (mW), written by the firmware. +pub(crate) const POLICY_GPU_POWER: usize = 0xad4; +/// Shared power-policy words of the firmware's primary performance limiter: the mask of active +/// limit reasons and the limit. The emergency reason with the limit at its floor is the clamp. +pub(crate) const POLICY_LIMITER_MASK: usize = 0x44; +pub(crate) const POLICY_LIMITER_LIMIT: usize = 0x48; +const LIMITER_EMERGENCY: u32 = 0x4; +const LIMITER_FLOOR: u32 = 100; +/// Control period while the GPU is busy. +pub(crate) const PERIOD_MS: u32 = 1000; +/// Restore the maximum after this long without a submission. +const IDLE_RESTORE_NS: u64 = 2_000_000_000; +/// Engage when the hotspot reaches this; release below it minus the hysteresis once the +/// controller has returned to its ceiling. +const ENGAGE_BELOW_TARGET_MC: i32 = 4000; +const RELEASE_HYSTERESIS_MC: i32 = 3000; +/// Lowest budget the governor sets (mW). +const FLOOR_MW: i64 = 10_000; +/// Lowest ceiling of the controller (mW): the GPU's draw at engage, but at least this. +const CEILING_MIN_MW: i64 = 40_000; +/// Proportional gain (mW per C) and integral gain (mW per C per second). Power follows the +/// budget within a second; the die settles over 30-60 s at ~1.2-1.5 C per W. +const KP_MW_PER_C: i64 = 1500; +const KI_MW_PER_C_S: i64 = 50; +/// Feed-forward on the hotspot's rise (mW per C/s), applied only while it rises, to curb the +/// overshoot when the GPU restarts from cool. +const KF_MW_PER_C_PER_S: i64 = 2000; +/// Without a converged integral, start at this fraction of the ceiling: the budget that holds +/// the target is about half the GPU's unconstrained draw. +const INITIAL_NUM: i64 = 3; +const INITIAL_DEN: i64 = 5; +/// The converged integral is kept for this long after the controller was last active, then +/// decays linearly to the cold-start integral over the decay time. +const MEMORY_HOLD_NS: u64 = 60_000_000_000; +const MEMORY_DECAY_NS: u64 = 300_000_000_000; +/// Effective target reduction per emergency clamp seen while governing, its limit, the lowest +/// effective target it leads to, and the time without a clamp after which it is reduced by one +/// step again. +const CLAMP_TARGET_STEP_MC: i32 = 1000; +const CLAMP_TARGET_MAX_MC: i32 = 6000; +const CLAMP_TARGET_FLOOR_MC: i32 = 84_000; +const CLAMP_TARGET_RECOVER_NS: u64 = 600_000_000_000; +/// Budget changes smaller than this are not published. +const STEP_MW: i64 = 250; + +/// CPU ceiling: one step lowers every cpufreq policy's maximum by this share of its hardware +/// maximum, down to the floor share. +const CPU_STEP_PERCENT: u32 = 4; +const CPU_FLOOR_PERCENT: u32 = 60; +/// Minimum time between CPU ceiling steps; the hotspot and the GPU draw follow within it. +const CPU_DWELL_NS: u64 = 2_000_000_000; +/// The GPU uses its budget when it draws at least this share of it, and leaves headroom below +/// the lower share (it then waits for the CPU). +const GPU_BOUND_PERCENT: i64 = 90; +const GPU_SLACK_PERCENT: i64 = 70; +/// The CPU's own heat: the hotspot this far above the target while the GPU budget is within the +/// slack of its floor. +const CPU_HEAT_MARGIN_MC: i32 = 2000; +const CPU_HEAT_FLOOR_SLACK_MW: i64 = 1000; + +fn enabled() -> bool { + crate::module_parameters::m5_thermal_governor.value() & 1 != 0 +} +fn cpu_balance_enabled() -> bool { + crate::module_parameters::m5_thermal_governor.value() & 4 != 0 +} +fn fan_boost_enabled() -> bool { + crate::module_parameters::m5_thermal_governor.value() & 8 != 0 +} +/// The proportional fan curve is on unless bit 4 (16) of m5_thermal_governor opts out. +fn fan_curve_enabled() -> bool { + crate::module_parameters::m5_thermal_governor.value() & 16 == 0 +} +pub(super) fn requested() -> bool { + enabled() +} +fn target_mc() -> i32 { + let parameter = crate::module_parameters::m5_thermal_target_mc.value(); + i32::try_from(parameter).unwrap_or(90_000).clamp(70_000, 104_000) +} + +/// Whether the firmware's primary limiter words describe the emergency clamp. +pub(crate) fn is_emergency_clamp(mask: u32, limit: u32) -> bool { + mask & LIMITER_EMERGENCY != 0 && limit <= LIMITER_FLOOR +} + +/// Pins the sensor provider and serializes reads with its probe and removal. +/// Its thermal zones are managed by the hwmon device registered during probe; +/// removal releases that device's resources while holding the provider lock. +struct SensorProvider(*mut bindings::device); + +impl SensorProvider { + fn acquire() -> Option { + // SAFETY: the bus and name have static lifetimes. The bus iterator + // returns an owned device reference on success. + let device = unsafe { + bindings::bus_find_device( + &bindings::platform_bus_type, + core::ptr::null_mut(), + c"macsmc-hwmon".as_ptr().cast(), + Some(bindings::device_match_name), + ) + }; + if device.is_null() { + return None; + } + // SAFETY: the owned reference keeps the device and mutex alive. No + // GPU driver lock is held during acquisition or sensor reads. + unsafe { bindings::mutex_lock(&raw mut (*device).mutex) }; + let provider = Self(device); + // SAFETY: the provider's lock protects binding and unbinding. + if unsafe { (*device).driver.is_null() } { + return None; + } + Some(provider) + } +} + +impl Drop for SensorProvider { + fn drop(&mut self) { + // SAFETY: acquire obtained this reference and locked this mutex on + // the current thread; the guard never leaves the sensor-read call. + unsafe { + bindings::mutex_unlock(&raw mut (*self.0).mutex); + bindings::put_device(self.0); + } + } +} + +/// The provider guard prevents its managed zones from disappearing during +/// lookup and read. Do not retain a zone pointer after dropping the guard. +fn zone_temperature_mc(_provider: &SensorProvider, name: &CStr) -> Option { + let mut temp: c_int = 0; + // SAFETY: name is a NUL-terminated string that outlives the call, and temp is a valid + // destination; the zone is looked up and read under the provider guard. + (unsafe { asahi_zone_temp(name.as_ptr(), &mut temp) } == 0) + .then_some(temp) + .filter(|temp| (0..=125_000).contains(temp)) +} + +/// SoC hotspot temperature in milli-C (the hottest GPU die sensor plus its offset when the hotspot +/// cannot be read), or None when no sensor can be read. Called without driver locks held: zone +/// lookups and sensor reads may sleep. +pub(crate) fn hotspot_temperature_mc() -> Option { + let provider = SensorProvider::acquire()?; + zone_temperature_mc(&provider, J714_HOTSPOT_ZONE) + .or_else(|| zone_temperature_mc(&provider, SOC_DIE_ZONE)) + .or_else(|| { + J714_GPU_DIE_ZONES.iter().filter_map(|name| zone_temperature_mc(&provider, name)).max() + .map(|temp| temp + GPU_DIE_TO_HOTSPOT_MC) + }) +} + +/// Keeps the fans boosted at least this long once boosted, so a short release or idle period +/// (a loading screen) does not cycle them. +const FAN_HOLD_NS: u64 = 30_000_000_000; + +/// Proportional fan curve on the hotspot while the GPU is busy: on J714 the SMC's automatic mode +/// keeps the fans near their minimum under sustained GPU load on Linux. From the board's curve +/// start ([`super::fw::m5::initdata::Board::fan_curve_start_mc`]) to the governor target the fans +/// ramp from the minimum to two thirds of the range, and on to FAN_CURVE_TOP_PERMILLE at +/// FAN_CURVE_TOP_MC. +static FAN_CURVE_START_MC: AtomicI32 = + AtomicI32::new(super::fw::m5::initdata::J714.fan_curve_start_mc); +const FAN_CURVE_TOP_MC: i32 = 98_000; +const FAN_CURVE_TARGET_PERMILLE: i32 = 660; +const FAN_CURVE_TOP_PERMILLE: i32 = 800; +/// Below this the curve releases the fans to automatic mode (after FAN_HOLD_NS). +const FAN_CURVE_RELEASE_MC: i32 = 82_000; +/// A new target is written only when it moves at least this far. +const FAN_CURVE_STEP_RPM: u32 = 150; + +/// Selects the board's thermal settings at probe; J714's until then. +pub(crate) fn select_board(board: &super::fw::m5::initdata::Board) { + FAN_CURVE_START_MC.store(board.fan_curve_start_mc, Ordering::Relaxed); +} + +/// The permille of the fan range the curve asks for at hotspot `temp_mc`. +fn fan_curve_permille(temp_mc: i32, target_mc: i32) -> i32 { + let start_mc = FAN_CURVE_START_MC.load(Ordering::Relaxed); + let target_mc = target_mc.max(start_mc + 1000); + if temp_mc <= start_mc { + 0 + } else if temp_mc <= target_mc { + (temp_mc - start_mc) * FAN_CURVE_TARGET_PERMILLE / (target_mc - start_mc) + } else { + let top = FAN_CURVE_TOP_MC.max(target_mc + 1000); + (FAN_CURVE_TARGET_PERMILLE + (temp_mc.min(top) - target_mc) + * (FAN_CURVE_TOP_PERMILLE - FAN_CURVE_TARGET_PERMILLE) / (top - target_mc)) + .min(FAN_CURVE_TOP_PERMILLE) + } +} + +/// Sets every fan the SMC hwmon cell lets a kernel policy set to `permille` of its +/// minimum-to-maximum range, or hands them back to the SMC for 0, under the provider lock (which +/// serializes with its unbinding). A fan whose target is within `min_step_rpm` of the new one +/// keeps it. Returns the first fan's target rpm, 0 once handed back. +fn request_fans(permille: u32, min_step_rpm: u32) -> Option { + let provider = SensorProvider::acquire()?; + // SAFETY: the provider lock keeps the device bound, as the request requires. + let rpm = unsafe { asahi_fans_request(provider.0.cast(), permille, min_step_rpm) }; + u32::try_from(rpm).ok() +} + +/// Fan boost: while the governor limits the GPU, every fan runs at its hardware maximum instead of +/// the SMC's automatic curve, which settles near two thirds of the maximum at the governor's +/// target. +/// Released (automatic mode) when the governor releases or the GPU goes idle, after a hold. +pub(crate) struct FanBoost { + boosted: bool, + since_ns: u64, + /// The fan target the curve last wrote (rpm), 0 while the fans are automatic. + curve_rpm: u32, +} + +impl FanBoost { + const fn new() -> Self { + Self { boosted: false, since_ns: 0, curve_rpm: 0 } + } + + /// Puts every fan in manual mode at `permille` of its minimum-to-maximum range, or back in + /// automatic mode for `None`. Returns the first fan's target rpm, 0 for automatic. + fn set_curve(&mut self, permille: Option) -> Option { + request_fans(permille.map_or(0, |permille| permille.clamp(1, 1000) as u32), 0) + } + + /// One governor step of the proportional curve: `busy` while the governor runs a busy GPU. + fn apply_curve(&mut self, busy: bool, temp_mc: Option, target_mc: i32, now_ns: u64) { + let permille = temp_mc.filter(|_| busy).map(|t| fan_curve_permille(t, target_mc)).unwrap_or(0); + if permille > 0 { + self.since_ns = now_ns; + // The target rpm moves with the range only when it moved enough. + if let Some(rpm) = self.set_curve_if_moved(permille) { + self.curve_rpm = rpm; + } + } else if self.curve_rpm != 0 + && (!busy || temp_mc.is_none_or(|t| t < FAN_CURVE_RELEASE_MC)) + && now_ns.saturating_sub(self.since_ns) >= FAN_HOLD_NS + && self.set_curve(None).is_some() + { + self.curve_rpm = 0; + } + } + + fn set_curve_if_moved(&mut self, permille: i32) -> Option { + // A fan already on the curve keeps its target while the new one is within a step. + let step = if self.curve_rpm != 0 { FAN_CURVE_STEP_RPM } else { 0 }; + request_fans(permille.clamp(1, 1000) as u32, step) + } + + fn set(&mut self, boost: bool) -> bool { + request_fans(if boost { 1000 } else { 0 }, 0).is_some() + } + + /// Applies one governor step's request. Called from the governor work without driver locks: + /// SMC accesses sleep. + fn apply(&mut self, want: bool, temp_mc: Option, now_ns: u64) { + if !fan_boost_enabled() { + if fan_curve_enabled() { + self.apply_curve(want, temp_mc, target_mc(), now_ns); + } else if self.curve_rpm != 0 && self.set_curve(None).is_some() { + self.curve_rpm = 0; + } + } + let want = want && fan_boost_enabled(); + if want { + self.since_ns = now_ns; + } + if want && !self.boosted { + self.boosted = self.set(true); + } else if !want && self.boosted + && (!fan_boost_enabled() || now_ns.saturating_sub(self.since_ns) >= FAN_HOLD_NS) + { + if self.set(false) { + self.boosted = false; + } + } + } + + pub(crate) fn release(&mut self) { + if self.boosted && self.set(false) { + self.boosted = false; + } + if self.curve_rpm != 0 && self.set_curve(None).is_some() { + self.curve_rpm = 0; + } + } + + pub(crate) fn boosted(&self) -> bool { + self.boosted || self.curve_rpm != 0 + } +} + +impl Drop for FanBoost { + fn drop(&mut self) { + self.release(); + } +} + +/// The host's thermal actuators besides the GPU budget, applied outside the device lock. +pub(crate) struct Actuators { + pub(crate) cpu: CpuCeiling, + pub(crate) fans: FanBoost, +} + +impl Actuators { + pub(crate) const fn new() -> Self { + Self { cpu: CpuCeiling::new(), fans: FanBoost::new() } + } + + pub(crate) fn apply(&mut self, advice: CpuAdvice, fans: bool, temp_mc: Option, now_ns: u64) { + self.cpu.apply(advice, now_ns); + self.fans.apply(fans, temp_mc, now_ns); + } + + pub(crate) fn release(&mut self) { + self.cpu.release(); + self.fans.release(); + } +} + +/// What the CPU ceiling should do after a governor step. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(crate) enum CpuAdvice { + /// Keep the current ceiling. + Hold, + /// The GPU draws its whole thermally limited budget: give it the CPU's share of the heat. + Lower, + /// The GPU is not limited by its budget (or waits for the CPU): give the CPU a step back. + Raise, + /// Not governing: drop the ceiling. + Release, +} + +/// One cpufreq policy's maximum-frequency request. +struct CpuRequest { + policy: *mut bindings::cpufreq_policy, + request: KBox, + max_khz: u32, +} + +// SAFETY: the policy reference and the request are only used from the governor work, under the +// ceiling's lock. +unsafe impl Send for CpuRequest {} + +/// Takes a counted reference on `cpu`'s cpufreq policy, or returns null where it has none. +#[cfg(CONFIG_CPU_FREQ)] +fn policy_get(cpu: u32) -> *mut bindings::cpufreq_policy { + // SAFETY: cpufreq_cpu_get accepts any CPU number and returns a counted reference or null. + unsafe { bindings::cpufreq_cpu_get(cpu) } +} + +/// Without cpufreq no CPU has a policy, so the ceiling never takes a request. +#[cfg(not(CONFIG_CPU_FREQ))] +fn policy_get(_cpu: u32) -> *mut bindings::cpufreq_policy { + core::ptr::null_mut() +} + +/// Drops a reference taken by [`policy_get`]. +/// +/// # Safety +/// +/// `policy` must hold a reference from [`policy_get`] that has not been dropped yet. +#[cfg(CONFIG_CPU_FREQ)] +unsafe fn policy_put(policy: *mut bindings::cpufreq_policy) { + // SAFETY: the caller holds the reference. + unsafe { bindings::cpufreq_cpu_put(policy) } +} + +/// Without cpufreq there is no reference to drop. +/// +/// # Safety +/// +/// None; unsafe only to match the cpufreq variant. +#[cfg(not(CONFIG_CPU_FREQ))] +unsafe fn policy_put(_policy: *mut bindings::cpufreq_policy) {} + +/// Shares the SoC's heat between the CPU and the GPU while the governor limits the GPU: when the +/// GPU is budget-bound it lowers the maximum +/// frequency of every CPU cluster one step at a time, and gives the steps back once the GPU no +/// longer uses its budget (it then waits for the CPU) or is no longer limited. The requests are +/// ordinary frequency QoS maxima, combined with any other limit on the policies. +pub(crate) struct CpuCeiling { + requests: KVec, + level: u32, + last_change_ns: u64, +} + +impl CpuCeiling { + pub(crate) const fn new() -> Self { + Self { requests: KVec::new(), level: 0, last_change_ns: 0 } + } + + fn max_level() -> u32 { + (100 - CPU_FLOOR_PERCENT) / CPU_STEP_PERCENT + } + + fn limit_khz(max_khz: u32, level: u32) -> i32 { + let share = 100 - level.min(Self::max_level()) * CPU_STEP_PERCENT; + (u64::from(max_khz) * u64::from(share) / 100).min(i32::MAX as u64) as i32 + } + + /// Takes a request on every policy (each policy once, through its first CPU). + fn acquire(&mut self) -> bool { + if !self.requests.is_empty() { + return true; + } + // SAFETY: nr_cpu_ids is set once during early boot. + let cpus = unsafe { bindings::nr_cpu_ids }; + for cpu in 0..cpus { + let policy = policy_get(cpu); + if policy.is_null() { + continue; + } + // SAFETY: the reference keeps the policy alive. + let (first, max_khz) = unsafe { ((*policy).cpu, (*policy).cpuinfo.max_freq) }; + let request = if first == cpu && max_khz != 0 { + // The zeroed (default) request is the unused state freq_qos_add_request expects. + KBox::new(bindings::freq_qos_request::default(), GFP_KERNEL).ok() + } else { + None + }; + let Some(mut request) = request.filter(|_| self.requests.reserve(1, GFP_KERNEL).is_ok()) + else { + // SAFETY: drops the reference taken above. + unsafe { policy_put(policy) }; + continue; + }; + // SAFETY: the policy outlives the request (the reference is held until removal) and + // the request is in its unused state, at a stable address in its box. + let added = unsafe { + bindings::freq_qos_add_request( + &raw mut (*policy).constraints, + &mut *request, + bindings::freq_qos_req_type_FREQ_QOS_MAX, + Self::limit_khz(max_khz, 0), + ) + }; + if added < 0 { + // SAFETY: drops the reference taken above. + unsafe { policy_put(policy) }; + continue; + } + // Capacity was reserved above, so this cannot fail. + let _ = self.requests.push(CpuRequest { policy, request, max_khz }, GFP_KERNEL); + } + !self.requests.is_empty() + } + + /// Removes every request and the policy references. + pub(crate) fn release(&mut self) { + for mut entry in self.requests.drain_all() { + // SAFETY: the request was added in acquire and the policy reference is still held. + unsafe { + bindings::freq_qos_remove_request(&mut *entry.request); + policy_put(entry.policy); + } + } + self.level = 0; + } + + fn publish(&mut self) { + for entry in self.requests.iter_mut() { + let limit = Self::limit_khz(entry.max_khz, self.level); + // SAFETY: the request was added in acquire and stays valid until release. + unsafe { bindings::freq_qos_update_request(&mut *entry.request, limit) }; + } + } + + /// Applies one governor step's advice. Called from the governor work without driver locks: + /// frequency QoS updates may sleep. + pub(crate) fn apply(&mut self, advice: CpuAdvice, now_ns: u64) { + if !cpu_balance_enabled() || advice == CpuAdvice::Release { + if !self.requests.is_empty() { + self.release(); + } + return; + } + if now_ns.saturating_sub(self.last_change_ns) < CPU_DWELL_NS { + return; + } + match advice { + CpuAdvice::Lower if self.level < Self::max_level() => { + if !self.acquire() { + return; + } + self.level += 1; + } + CpuAdvice::Raise if self.level > 0 => { + self.level -= 1; + if self.level == 0 { + self.release(); + self.last_change_ns = now_ns; + return; + } + } + _ => return, + } + self.last_change_ns = now_ns; + self.publish(); + } +} + +impl Drop for CpuCeiling { + fn drop(&mut self) { + self.release(); + } +} + +pub(crate) struct Governor { + maximum_mw: u32, + budget_mw: u32, + integral_mw: i64, + /// Upper limit of the controller while active: the GPU's draw at engage. + ceiling_mw: i64, + /// Integral the controller last converged to (output inside its range), kept across releases. + converged_mw: i64, + active: bool, + last_tick_ns: u64, + last_activity_ns: u64, + /// Last tick at which the controller was active, for the memory decay. + last_active_ns: u64, + last_temp_mc: Option, + /// Effective target reduction learnt from emergency clamps, and when it last changed. + target_offset_mc: i32, + target_offset_ns: u64, + was_clamped: bool, +} + +impl Governor { + pub(crate) const fn new() -> Self { + Self { maximum_mw: 0, budget_mw: 0, integral_mw: 0, ceiling_mw: 0, converged_mw: 0, active: false, + last_tick_ns: 0, last_activity_ns: 0, last_active_ns: 0, last_temp_mc: None, target_offset_mc: 0, + target_offset_ns: 0, was_clamped: false } + } + + /// The firmware accepted a submission; returns whether the control loop should run. + pub(crate) fn note_activity(&mut self, now_ns: u64) -> bool { + self.last_activity_ns = now_ns; + requested() || self.budget_mw != self.maximum_mw + } + + /// Learn the calibrated maximum from the budget word before the first change. + pub(crate) fn set_maximum(&mut self, current_mw: u32) { + if self.maximum_mw == 0 && i64::from(current_mw) >= FLOOR_MW { + self.maximum_mw = current_mw; + self.budget_mw = current_mw; + } + } + + /// The budget to publish while stopping, if it differs from the maximum. + pub(crate) fn restore(&mut self) -> Option { + self.active = false; + self.integral_mw = 0; + self.converged_mw = 0; + self.target_offset_mc = 0; + if self.maximum_mw == 0 || self.budget_mw == self.maximum_mw { + return None; + } + self.budget_mw = self.maximum_mw; + Some(self.budget_mw) + } + + /// Integral to engage with: the converged one while it is fresh, decaying to the cold-start + /// integral (a fraction of the ceiling) after a long period without control. + fn resume_integral(&self, now_ns: u64) -> i64 { + let initial = self.ceiling_mw * INITIAL_NUM / INITIAL_DEN; + if self.converged_mw == 0 { + return initial; + } + let kept = self.converged_mw.min(self.ceiling_mw); + let cool_ns = now_ns.saturating_sub(self.last_active_ns); + if cool_ns <= MEMORY_HOLD_NS { + return kept; + } + let decay_ns = (cool_ns - MEMORY_HOLD_NS).min(MEMORY_DECAY_NS); + kept + (initial - kept) * (decay_ns / 1_000_000) as i64 / (MEMORY_DECAY_NS / 1_000_000) as i64 + } + + /// Learn from the firmware's emergency clamp: lower the effective target by a step on each + /// clamp seen while governing, raise it again after a long period without one. + fn adapt_target(&mut self, clamped: bool, now_ns: u64) { + if clamped && !self.was_clamped && self.active { + let limit = (target_mc() - CLAMP_TARGET_FLOOR_MC).clamp(0, CLAMP_TARGET_MAX_MC); + self.target_offset_mc = (self.target_offset_mc + CLAMP_TARGET_STEP_MC).min(limit); + self.target_offset_ns = now_ns; + } else if !clamped && self.target_offset_mc > 0 + && now_ns.saturating_sub(self.target_offset_ns) >= CLAMP_TARGET_RECOVER_NS + { + self.target_offset_mc -= CLAMP_TARGET_STEP_MC; + self.target_offset_ns = now_ns; + } + self.was_clamped = clamped; + } + + /// How the CPU ceiling should follow the last step, from the GPU's draw (mW) against the + /// budget it was given. While governing, a GPU that draws its whole budget below its ceiling + /// is limited by heat the CPU shares; one that leaves headroom waits for the CPU. When the + /// hotspot stays above the target with the GPU budget at its floor, the heat is the CPU's + /// and only the CPU can keep the firmware from clamping the GPU. + pub(crate) fn cpu_advice(&self, power_mw: u32, now_ns: u64) -> CpuAdvice { + if self.maximum_mw == 0 || now_ns.saturating_sub(self.last_activity_ns) >= IDLE_RESTORE_NS { + return CpuAdvice::Release; + } + if !self.active { + // Released by the controller: hand the steps back one at a time, so the CPU's heat + // returns gradually and does not re-engage the governor at once. + return CpuAdvice::Raise; + } + let budget = i64::from(self.budget_mw); + let power = i64::from(power_mw); + let target = target_mc() - self.target_offset_mc; + let temp = self.last_temp_mc.unwrap_or(target); + if temp >= target + CPU_HEAT_MARGIN_MC + && budget <= FLOOR_MW.min(self.ceiling_mw) + CPU_HEAT_FLOOR_SLACK_MW + { + CpuAdvice::Lower + } else if budget < self.ceiling_mw && power * 100 >= budget * GPU_BOUND_PERCENT { + CpuAdvice::Lower + } else if budget >= self.ceiling_mw + || (power * 100 < budget * GPU_SLACK_PERCENT && temp < target) + { + // A GPU waiting for the CPU gets it back only below the target: above it, the + // controller is still taking heat out. + CpuAdvice::Raise + } else { + CpuAdvice::Hold + } + } + + /// Whether the fans should be boosted: while the controller governs a busy GPU. + pub(crate) fn fans_wanted(&self, now_ns: u64) -> bool { + self.maximum_mw != 0 && self.active + && now_ns.saturating_sub(self.last_activity_ns) < IDLE_RESTORE_NS + } + + /// One control step with the hotspot temperature, the GPU's current draw (mW) and whether the + /// firmware's emergency clamp is active. Returns the budget to publish (if it changed) and + /// whether to run again. + pub(crate) fn tick(&mut self, temp_mc: Option, power_mw: u32, clamped: bool, now_ns: u64) + -> (Option, bool) + { + let maximum = i64::from(self.maximum_mw); + let idle = now_ns.saturating_sub(self.last_activity_ns) >= IDLE_RESTORE_NS; + let dt_ns = now_ns.saturating_sub(self.last_tick_ns).min(2 * IDLE_RESTORE_NS); + self.last_tick_ns = now_ns; + let previous_mc = core::mem::replace(&mut self.last_temp_mc, temp_mc); + if enabled() { + self.adapt_target(clamped, now_ns); + } + let target = target_mc() - self.target_offset_mc; + let desired = match temp_mc { + _ if maximum == 0 => return (None, false), + None => { + self.active = false; + maximum + } + _ if idle || !enabled() => { + self.active = false; + maximum + } + Some(temp) => { + let error_mc = i64::from(target - temp); + let proportional = KP_MW_PER_C * error_mc / 1000; + let rise_mc_per_s = match previous_mc { + Some(previous) if temp > previous && dt_ns >= 1_000_000 => + i64::from(temp - previous) * 1000 / (dt_ns / 1_000_000) as i64, + _ => 0, + }; + let feed_forward = KF_MW_PER_C_PER_S * rise_mc_per_s / 1000; + if !self.active && temp >= target - ENGAGE_BELOW_TARGET_MC { + self.active = true; + self.ceiling_mw = i64::from(power_mw).max(CEILING_MIN_MW).clamp(FLOOR_MW, maximum); + self.integral_mw = self.resume_integral(now_ns); + } + if self.active && clamped { + // The firmware's emergency clamp: hold the budget and the integral until it + // ends, so the GPU resumes at the converged budget. + self.last_active_ns = now_ns; + i64::from(self.budget_mw) + } else if self.active { + self.last_active_ns = now_ns; + let floor = FLOOR_MW.min(self.ceiling_mw); + let candidate = self.integral_mw + KI_MW_PER_C_S * error_mc * (dt_ns / 1_000_000) as i64 / 1_000_000; + let output = proportional + candidate - feed_forward; + // Anti-windup: integrate only while the output is inside its range; at a + // limit, back-calculate the integral so the output sits on the limit. + let output = if output > self.ceiling_mw { + self.integral_mw = self.ceiling_mw - proportional + feed_forward; + self.ceiling_mw + } else if output < floor { + self.integral_mw = floor - proportional + feed_forward; + floor + } else { + self.integral_mw = candidate; + self.converged_mw = candidate; + output + }; + if output >= self.ceiling_mw + && temp < target - ENGAGE_BELOW_TARGET_MC - RELEASE_HYSTERESIS_MC + { + // Released: back to the calibrated maximum. + self.active = false; + maximum + } else { + output + } + } else { + maximum + } + } + }; + let desired = if desired >= maximum { maximum } else { desired - desired % STEP_MW }; + let publish = (desired != i64::from(self.budget_mw)).then(|| { + self.budget_mw = desired as u32; + self.budget_mw + }); + let again = (!idle && requested()) || self.budget_mw != self.maximum_mw; + (publish, again) + } +} diff --git a/drivers/gpu/drm/asahi/g17/timeout.rs b/drivers/gpu/drm/asahi/g17/timeout.rs index ce003e6b18196..3da5e255408b5 100644 --- a/drivers/gpu/drm/asahi/g17/timeout.rs +++ b/drivers/gpu/drm/asahi/g17/timeout.rs @@ -157,6 +157,47 @@ pub(crate) fn attribute(own: Head, renewals: u8, others: impl Iterator bool { + use core::sync::atomic::Ordering; + let now = crate::g17::now_ns(); + if PROGRESS.swap(word, Ordering::Relaxed) != word { + LAST_PROGRESS_NS.store(now, Ordering::Relaxed); + } + match LAST_PROGRESS_NS.load(Ordering::Relaxed) { + 0 => false, + last => now.saturating_sub(last) < PROGRESS_INTERVAL_NS, + } +} + +/// Work whose context has an older node in use ahead of its own is not late: the firmware +/// lists a context's nodes by their words and flushes from the head, so this work's stamps wait +/// behind the older node until it drains. A published job behind its context's older node is +/// renewed while the firmware's progress word moves, with two limits: +/// +/// - Not once its VM has failed work (`failed`). The older node's holder may be that failed +/// work, held until the firmware drains it, and a failed blocker excuses nothing ([`decide`]). +/// - At most [`SHARED_RENEWALS`] times, counted with the job's other deferrals (`renewals`). +/// The older node's own job runs under the same allowances and started earlier, so a healthy +/// older node drains or fails within that budget; past it the job times out as any other. +/// +/// A firmware that stops progressing for the interval expires the job as before, and an older +/// node of another VM still goes through [`attribute`]. +pub(crate) fn behind_own_head(published: bool, older: Option, progressing: bool, + failed: bool, renewals: u8) -> bool { + published && older.is_some() && progressing && !failed && renewals < SHARED_RENEWALS +} + #[cfg_attr(not(test), kernel::macros::kunit_tests(asahi_g17_execution_allowance))] mod tests { use super::*; @@ -284,6 +325,19 @@ mod tests { assert!(!renew_shared(true, true, &mut renewals)); } #[test] + fn work_behind_its_own_context_is_renewed_only_while_healthy_and_bounded() { + let older = Some(7); + for renewals in 0..SHARED_RENEWALS { + assert!(behind_own_head(true, older, true, false, renewals)); + } + assert!(!behind_own_head(true, older, true, false, SHARED_RENEWALS)); + assert!(!behind_own_head(true, older, true, false, u8::MAX)); + assert!(!behind_own_head(true, older, true, true, 0)); + assert!(!behind_own_head(true, None, true, false, 0)); + assert!(!behind_own_head(true, older, false, false, 0)); + assert!(!behind_own_head(false, older, true, false, 0)); + } + #[test] fn exhausted_or_invalid_credit_never_wraps() { for value in 4..=u8::MAX { let mut renewals = value; diff --git a/drivers/gpu/drm/asahi/g18/boot/mod.rs b/drivers/gpu/drm/asahi/g18/boot/mod.rs index 7cb051ec680f6..28717fe4a261f 100644 --- a/drivers/gpu/drm/asahi/g18/boot/mod.rs +++ b/drivers/gpu/drm/asahi/g18/boot/mod.rs @@ -171,6 +171,8 @@ pub(crate) const fn stage_runs_firmware(stage: u32) -> bool { /// The firmware initializes once per boot; the module is pinned from then on, /// so this survives unbind/rebind and refuses a second attempt. static G18_FIRMWARE_ATTEMPTED: AtomicBool = AtomicBool::new(false); +/// Set once the module is pinned for the retained session. +static G18_MODULE_RETAINED: AtomicBool = AtomicBool::new(false); /// Bit per PSD request kind written by this module instance: bit 0 sgx, /// bit 1 gfx-asc, bit 2 gfx1-asc (power-on), bit 3 restore. Each kind is /// written at most once per module load; a re-bind does not repeat it. @@ -2023,13 +2025,10 @@ fn prepare_firmware_session(pdev: &platform::Device>, probe: T8152Probe // Point of no return: from here every path retains the session, and the // module stays loaded so RTKit callbacks keep valid code until reboot. + // No ASC CPU has started yet: once module removal has begun, refuse with + // nothing started and the session freed. + crate::retain::retain_module(&G18_MODULE_RETAINED)?; G18_FIRMWARE_ATTEMPTED.store(true, Ordering::Release); - // SAFETY: THIS_MODULE is this module; the reference is never dropped. - unsafe { - bindings::__module_get( - ::THIS_MODULE.as_ptr(), - ) - }; Ok(Retained(Some(session))) } diff --git a/drivers/gpu/drm/asahi/gem.rs b/drivers/gpu/drm/asahi/gem.rs index 551e801ba2a29..60e4827957427 100644 --- a/drivers/gpu/drm/asahi/gem.rs +++ b/drivers/gpu/drm/asahi/gem.rs @@ -236,7 +236,7 @@ pub(crate) struct AsahiObjConfig { /// Create a new kernel-owned GEM object. pub(crate) fn new_kernel_object(dev: &AsahiDrmDevice, size: usize) -> Result { - new_kernel_object_mapped(dev, size, false) + new_kernel_object_mapped(dev, size, false, true) } /// Create a new kernel-owned GEM object whose kernel mapping is write-combined. @@ -244,7 +244,17 @@ pub(crate) fn new_kernel_object(dev: &AsahiDrmDevice, size: usize) -> Result Result { - new_kernel_object_mapped(dev, size, true) + new_kernel_object_mapped(dev, size, true, true) +} + +/// Create a new kernel-owned write-combined GEM object that is not counted as kernel memory: +/// its owner holds a user charge for it ([`memacct::Charge::user`]), and counting both would +/// take its size twice from the device limit. +pub(crate) fn new_kernel_object_wc_user_charged( + dev: &AsahiDrmDevice, + size: usize, +) -> Result { + new_kernel_object_mapped(dev, size, true, false) } /// Validate before rounding: wrapping a user size can create a much smaller @@ -259,8 +269,18 @@ pub(crate) fn checked_object_size(size: usize) -> Result { .ok_or(EOVERFLOW) } -fn new_kernel_object_mapped(dev: &AsahiDrmDevice, size: usize, map_wc: bool) -> Result { +fn new_kernel_object_mapped( + dev: &AsahiDrmDevice, + size: usize, + map_wc: bool, + kernel_charge: bool, +) -> Result { let size = checked_object_size(size)?; + let charge = if kernel_charge { + memacct::Charge::kernel(&memacct::LEDGER, size as u64) + } else { + memacct::Charge::none() + }; let gem = shmem::Object::::new( dev, size, @@ -274,7 +294,7 @@ fn new_kernel_object_mapped(dev: &AsahiDrmDevice, size: usize, map_wc: bool) -> flags: 0, exportable: false, kernel: true, - charge: memacct::Charge::kernel(&memacct::LEDGER, size as u64), + charge, }, )?; diff --git a/drivers/gpu/drm/asahi/hw/mod.rs b/drivers/gpu/drm/asahi/hw/mod.rs index bf1fd92822ced..94f8f11e81bb4 100644 --- a/drivers/gpu/drm/asahi/hw/mod.rs +++ b/drivers/gpu/drm/asahi/hw/mod.rs @@ -20,6 +20,7 @@ mod t604x; pub(crate) mod t8103; pub(crate) mod t8112; pub(crate) mod t8140; +pub(crate) mod t6050; #[cfg(CONFIG_DRM_ASAHI_G18)] pub(crate) mod t8152; diff --git a/drivers/gpu/drm/asahi/hw/t6040.rs b/drivers/gpu/drm/asahi/hw/t6040.rs index 5f6b9944a811e..102e8d813a83f 100644 --- a/drivers/gpu/drm/asahi/hw/t6040.rs +++ b/drivers/gpu/drm/asahi/hw/t6040.rs @@ -33,13 +33,14 @@ pub(crate) const CONFIG: Config = Config { crash_buffers: CrashBuffers::FirmwareData, // Read by the firmware through the kernel lower root, whose top-level entries it only // samples when it starts. - pm_metrics: FixedBuffer { + profile: crate::g17::FirmwareProfile::PublishedRoots, + pm_metrics: Some(FixedBuffer { va: 0x10_0008_0000, size: 0x8000, - }, + }), windows: ®ISTER_WINDOWS, reserved_slots: &[], - perf: &PERF, + perf: Some(&PERF), opps: OPPS, // The no-map tables are needed until the boot loader adds the UAT ranges to /memory. The // rest is the policy the M4 Pro has run with; none of it was tested otherwise. @@ -49,10 +50,12 @@ pub(crate) const CONFIG: Config = Config { page_tlbi: true, handoff_lock: false, handoff_slot: 0xffff_ffff, + handoff_unk2: 0, native_gpu_permissions: true, }, - region_c: ®ION_C, + region_c: Some(®ION_C), chip: None, + m5: None, adopt_root_entries: false, fabric_bridge: false, // The firmware's per-slot recovery keys are not known for this build; recovery relies on @@ -69,7 +72,8 @@ pub(crate) const CONFIG: Config = Config { name: "Apple GFX 25G83 for T6040 (normalized loaded image)", text_size: 0x60000, data_size: 0x138000, - entropy: [0x59839, 0x5991c], + entropy: &[[0x59839, 0x5991c]], + boot_filled: &[], text_sha256: [ 0x03, 0xdf, 0x75, 0x48, 0x4a, 0x7f, 0xdd, 0x6e, 0x4b, 0x47, 0x7f, 0x95, 0x6c, 0xaa, 0x5b, 0x44, 0x06, 0xdf, 0xff, 0x4d, 0x19, 0x17, 0x28, 0x62, 0x7f, 0x53, @@ -80,7 +84,8 @@ pub(crate) const CONFIG: Config = Config { name: "Apple GFX 25G83 for T6040 (normalized loaded image, alternate J616s fill)", text_size: 0x60000, data_size: 0x138000, - entropy: [0x59839, 0x5991c], + entropy: &[[0x59839, 0x5991c]], + boot_filled: &[], text_sha256: [ 0x19, 0x0c, 0xef, 0x19, 0x46, 0xa2, 0x3d, 0xfb, 0xde, 0x18, 0x05, 0x92, 0x62, 0x96, 0x31, 0xc3, 0xb2, 0x8d, 0x05, 0x09, 0xa1, 0x7e, 0xb7, 0x61, 0x08, 0xb7, @@ -91,7 +96,8 @@ pub(crate) const CONFIG: Config = Config { name: "Apple GFX 25G83 for T6040 (normalized loaded image, second alternate J616s fill)", text_size: 0x60000, data_size: 0x138000, - entropy: [0x59839, 0x5991c], + entropy: &[[0x59839, 0x5991c]], + boot_filled: &[], text_sha256: [ 0xd7, 0x25, 0x26, 0x4d, 0x48, 0x31, 0xb9, 0x5a, 0x5b, 0x8a, 0x37, 0x5a, 0xbf, 0x2f, 0xb5, 0xcd, 0xfb, 0xd6, 0xd7, 0xcb, 0xc7, 0xf1, 0xba, 0xa2, 0xcc, 0x8b, diff --git a/drivers/gpu/drm/asahi/hw/t6041.rs b/drivers/gpu/drm/asahi/hw/t6041.rs index 11385a7a0a4b2..9d61935a60807 100644 --- a/drivers/gpu/drm/asahi/hw/t6041.rs +++ b/drivers/gpu/drm/asahi/hw/t6041.rs @@ -26,13 +26,14 @@ pub(super) const BASE: Config = Config { secondary_root_offset: None, crash_buffers: CrashBuffers::FirmwareData, // Unverified placement; use the M4 Pro layout. - pm_metrics: FixedBuffer { + profile: crate::g17::FirmwareProfile::PublishedRoots, + pm_metrics: Some(FixedBuffer { va: 0x10_0008_0000, size: 0x8000, - }, + }), windows: ®ISTER_WINDOWS, reserved_slots: &RESERVED_SLOTS, - perf: &PERF, + perf: Some(&PERF), opps: OPPS, uat_quirks: UatQuirks { reserved_tables: true, @@ -40,10 +41,12 @@ pub(super) const BASE: Config = Config { page_tlbi: true, handoff_lock: false, handoff_slot: 0xffff_ffff, + handoff_unk2: 0, native_gpu_permissions: true, }, - region_c: ®ION_C, + region_c: Some(®ION_C), chip: Some(&CHIP), + m5: None, adopt_root_entries: false, fabric_bridge: false, recovery_slot_keys: None, @@ -87,7 +90,8 @@ const BUILD_25F84: fw::variant::Build = fw::variant::Build { name: "Apple GFX 25F84 for T6041 (normalized loaded image)", text_size: 0x60000, data_size: 0x138000, - entropy: [0x5983a, 0x5991d], + entropy: &[[0x5983a, 0x5991d]], + boot_filled: &[], text_sha256: [ 0x6d, 0xa4, 0xd4, 0x42, 0x35, 0x35, 0x14, 0x44, 0xf6, 0xec, 0x00, 0x33, 0xf1, 0x9c, 0xbf, 0x98, 0xd3, 0x43, 0x91, 0x28, 0x83, 0xe4, 0xe2, 0x94, 0xc1, 0x8b, 0xb8, 0xb8, @@ -114,7 +118,8 @@ pub(super) const BUILD_25G83: fw::variant::Build = fw::variant::Build { name: "Apple GFX 25G83 for T6041 (normalized loaded image, 32-core J616C)", text_size: 0x60000, data_size: 0x138000, - entropy: [0x59839, 0x5991c], + entropy: &[[0x59839, 0x5991c]], + boot_filled: &[], text_sha256: [ 0xaa, 0x11, 0x3e, 0xd6, 0x20, 0x79, 0xbc, 0xfd, 0xee, 0xf0, 0xf8, 0x6c, 0x2a, 0x8d, 0x7e, 0xae, 0x9f, 0x90, 0x7b, 0x03, 0xbe, 0x28, 0xcc, 0xf2, 0x98, 0xc9, 0x43, 0x95, @@ -125,7 +130,8 @@ pub(super) const BUILD_25G83: fw::variant::Build = fw::variant::Build { name: "Apple GFX 25G83 for T6041 (normalized loaded image, 40-core J614C)", text_size: 0x60000, data_size: 0x138000, - entropy: [0x59839, 0x5991c], + entropy: &[[0x59839, 0x5991c]], + boot_filled: &[], text_sha256: [ 0x7c, 0x04, 0x36, 0x0e, 0x05, 0xa6, 0xb8, 0x62, 0x8f, 0x61, 0x26, 0x2d, 0x0a, 0x24, 0x41, 0x66, 0x18, 0x7a, 0xd8, 0x7a, 0x73, 0x47, 0x16, 0xd8, 0x91, 0x09, 0x08, 0xe9, diff --git a/drivers/gpu/drm/asahi/hw/t6041_25g83.rs b/drivers/gpu/drm/asahi/hw/t6041_25g83.rs index 370561d906bf2..e23b419364a78 100644 --- a/drivers/gpu/drm/asahi/hw/t6041_25g83.rs +++ b/drivers/gpu/drm/asahi/hw/t6041_25g83.rs @@ -44,8 +44,8 @@ use crate::g17::{Config, CoreLayout}; pub(super) const CONFIG: Config = Config { windows: ®ISTER_WINDOWS, reserved_slots: &[], - perf: &PERF, - region_c: ®ION_C, + perf: Some(&PERF), + region_c: Some(®ION_C), chip: Some(&CHIP), firmware: fw::variant::Firmware::Identified(&[t6041::BUILD_25G83]), core_layout: CoreLayout { diff --git a/drivers/gpu/drm/asahi/hw/t6050.rs b/drivers/gpu/drm/asahi/hw/t6050.rs new file mode 100644 index 0000000000000..ae2b48d9184d1 --- /dev/null +++ b/drivers/gpu/drm/asahi/hw/t6050.rs @@ -0,0 +1,250 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! M5 Pro, M5 Max and M5 Ultra GPU firmware and address-space configuration. +//! +//! The M5 firmware builds its HW data, performance tables and region C in its own graph +//! ([`crate::g17::fw::m5::initdata`]), so the 26A428/25G83 graph tables of [`Config`] are empty. +use crate::g17::{Config, CoreLayout, CrashBuffers, FirmwareProfile, Identity}; +use crate::hw::{GpuGen, GpuVariant}; +use crate::regs::G17Id; + +/// M5 Pro (J714s): ID variant 3, GPU variant S, on bare metal like the M5 Max. +pub(crate) const CONFIG: Config = Config { + chip_id: 0x6050, + identity: Identity { + family: 0xb, + variant: 3, + num_dies: 1, + gpu_variant: GpuVariant::S, + usc_generation: 3, + hal_generation: 300, + }, + uat_ias: 42, + uat_oas: 42, + // Both roles publish their kernel roots. The boot arguments iBoot writes into each role's + // TEXT give the primary's tables at `pagetables` (BPTP, the ADT gfx-shared-region) and the + // secondary's 0x40000 above it, 0x40000 each (SPTP). + secondary_root_offset: Some(0x40000), + // Each role places its own buffers. Its crash log goes in its __DATA_SHARED_RO, inside + // `fw-data`, as on the M5 Max, whose data segments have the same layout. + crash_buffers: CrashBuffers::FirmwareData, + profile: FirmwareProfile::M5PublishedRoots, + pm_metrics: None, + windows: &[], + reserved_slots: &[], + perf: None, + opps: 0, + // The M5 Max and Ultra's UAT policy, for the same firmware family. + uat_quirks: CONFIG_ULTRA.uat_quirks, + region_c: None, + chip: None, + m5: Some(&crate::g17::fw::m5::initdata::J714), + adopt_root_entries: true, + fabric_bridge: false, + recovery_slot_keys: None, + gpu_generation: GpuGen::G17, + // The firmware in `fw-text` must be this build. iBoot loads the G17S build of both roles + // back to back: the primary's TEXT 0x50000, then the secondary's 0x54000, with the same + // entropy records and boot-filled ranges as the M5 Max. The digest is that of the + // firmware's TEXT with those ranges zeroed, measured on two J714s boots. + firmware: crate::g17::fw::variant::Firmware::Identified(&[crate::g17::fw::variant::Build { + os_version: [27, 0, 0], + name: "27.0 (26A428), G17S", + images: &[ + crate::fwid::FirmwareBuild { + name: "26A428 for T6050 G17S, primary then secondary", + text_size: 0xa4000, + data_size: 0x184000, + entropy: &[[0x49e60, 0x49f43], [0x9d918, 0x9d9fb]], + boot_filled: &[ + (0x224, 4), (0x23c, 8), (0x49720, 0x740), (0x49e60, 0x231), + (0x50224, 4), (0x5023c, 8), (0x9d1d8, 0x740), (0x9d918, 0x231), + ], + text_sha256: [ + 0x85, 0xd0, 0xfd, 0x82, 0x80, 0xf7, 0x24, 0x65, 0x5c, 0xa9, 0xa9, 0x71, 0x7b, + 0xc9, 0x1a, 0x8a, 0x5e, 0xf6, 0x09, 0x28, 0xd7, 0x72, 0x00, 0xf6, 0xd5, 0x3b, + 0xd0, 0x99, 0xb9, 0x47, 0x39, 0xc6, + ], + }, + ], + variant: &crate::g17::fw::variant::V26A428_M5, + config: None, + }]), + // Two clusters of ten cores. The buffers are sized for every core slot of the board, so the + // board's cluster and slot counts admit the GPU, whichever cores are fused off. + core_layout: CoreLayout { + cores: None, + alternate_core_masks: None, + operand_state: crate::g17::fw::compute::OperandState::TWO_BY_TEN, + deflake: crate::g17::fw::render::Deflake::TWO_CLUSTERS, + compute_blocks: crate::g17::fw::freelist::COMPUTE_BLOCKS, + }, +}; + +/// M5 Max: ID variant 4, GPU variant C, HAL generation 300 like the M5 Pro. +pub(crate) const CONFIG_MAX: Config = Config { + chip_id: 0x6050, + identity: Identity { + family: 0xb, + variant: 4, + num_dies: 1, + gpu_variant: GpuVariant::C, + usc_generation: 3, + hal_generation: 300, + }, + uat_ias: 42, + uat_oas: 42, + // Bare metal, as the M5 Ultra: both roles publish their kernel roots, the secondary's + // 0x40000 into `pagetables`, and each asks for its crash log inside `fw-data`. + secondary_root_offset: Some(0x40000), + crash_buffers: CrashBuffers::FirmwareData, + profile: FirmwareProfile::M5PublishedRoots, + pm_metrics: None, + windows: &[], + reserved_slots: &[], + perf: None, + opps: 0, + // The M5 Ultra's UAT policy, which this board's firmware expects too. + uat_quirks: CONFIG_ULTRA.uat_quirks, + region_c: None, + chip: None, + m5: Some(&crate::g17::fw::m5::initdata::J716C), + adopt_root_entries: true, + fabric_bridge: false, + recovery_slot_keys: None, + gpu_generation: GpuGen::G17, + // As on the M5 Ultra, the firmware in `fw-text` must be this build. The boot loader loads + // the G17C build of both roles back to back: the primary's TEXT 0x50000, then the + // secondary's 0x54000. The digest is that of the firmware's TEXT with the boot-filled ranges + // zeroed. + firmware: crate::g17::fw::variant::Firmware::Identified(&[crate::g17::fw::variant::Build { + os_version: [27, 0, 0], + name: "27.0 (26A428), G17C", + images: &[ + crate::fwid::FirmwareBuild { + name: "26A428 for T6050 G17C, primary then secondary", + text_size: 0xa4000, + data_size: 0x184000, + entropy: &[[0x49e60, 0x49f43], [0x9d918, 0x9d9fb]], + boot_filled: &[ + (0x224, 4), (0x23c, 8), (0x49720, 0x740), (0x49e60, 0x231), + (0x50224, 4), (0x5023c, 8), (0x9d1d8, 0x740), (0x9d918, 0x231), + ], + text_sha256: [ + 0x30, 0x4d, 0x85, 0x44, 0x66, 0x29, 0xbc, 0xdc, 0xa5, 0x95, 0x80, 0xb5, 0xff, + 0x00, 0x6d, 0xd8, 0x9f, 0x2c, 0xd8, 0xb4, 0xca, 0x60, 0x59, 0x9a, 0x0c, 0xb5, + 0x5c, 0x35, 0xec, 0xb0, 0x98, 0xf4, + ], + }, + ], + variant: &crate::g17::fw::variant::V26A428_M5, + config: None, + }]), + // Four clusters of ten cores. The buffers are sized for every core slot of the board, so the + // board's cluster and slot counts admit the GPU, whichever cores are fused off. + core_layout: CoreLayout { + cores: None, + alternate_core_masks: None, + operand_state: crate::g17::fw::compute::OperandState::FOUR_BY_TEN, + deflake: crate::g17::fw::render::Deflake::FOUR_CLUSTERS, + compute_blocks: crate::g17::fw::freelist::COMPUTE_BLOCKS, + }, +}; + +/// M5 Ultra (J775D, two dies, G17D) on bare metal: both roles publish their kernel roots, the +/// secondary's 0x40000 into `pagetables`, and each asks for its crash log inside `fw-data`. The +/// UAT tables are no-map reservations outside /memory; the rest of the UAT policy is the one +/// this board has run with: every table change written back, page-by-page TLB invalidation, and +/// the handoff values its firmware expects (current slot 0xffff, the byte after the flush records +/// 1, the lock never taken). +pub(crate) const CONFIG_ULTRA: Config = Config { + chip_id: 0x6050, + // Family 0xb (G17), variant 6 (G17D), revision 0x11. + identity: Identity { + family: 0xb, + variant: 6, + num_dies: 2, + gpu_variant: GpuVariant::D, + usc_generation: 3, + hal_generation: 300, + }, + uat_ias: 42, + uat_oas: 42, + secondary_root_offset: Some(0x40000), + crash_buffers: CrashBuffers::FirmwareData, + profile: FirmwareProfile::M5PublishedRoots, + pm_metrics: None, + windows: &[], + reserved_slots: &[], + perf: None, + opps: 0, + uat_quirks: crate::mmu::UatQuirks { + reserved_tables: true, + clean_tables: true, + page_tlbi: true, + handoff_lock: false, + handoff_slot: 0xffff, + handoff_unk2: 1, + native_gpu_permissions: false, + }, + region_c: None, + chip: None, + m5: Some(&crate::g17::fw::m5::initdata::J775D), + adopt_root_entries: true, + fabric_bridge: false, + recovery_slot_keys: None, + gpu_generation: GpuGen::G17, + // On bare metal nothing vouches for the firmware the boot loader left in `fw-text`: it + // must be this build, or the GPU is refused before anything is written to it. + firmware: crate::g17::fw::variant::Firmware::Identified(&[crate::g17::fw::variant::Build { + os_version: [27, 0, 0], + name: "27.0 (26A428)", + images: &[crate::fwid::FirmwareBuild { + // The boot loader loads both roles' TEXT back to back into `fw-text`: the primary's + // 0x50000, then the secondary's 0x58000, each with its own GKTS/ECAP entropy + // records. The digest is that of the firmware's TEXT with the boot-filled ranges + // zeroed, so a loaded copy normalizes to it on every boot. + name: "26A428 for T6050, primary then secondary", + text_size: 0xa8000, + data_size: 0x184000, + entropy: &[[0x49e60, 0x49f43], [0xa1918, 0xa19fb]], + // Per role: two load fields of the image header, then the two sections the boot + // loader fills with the boot arguments (the entropy included). + boot_filled: &[ + (0x224, 4), (0x23c, 8), (0x49720, 0x740), (0x49e60, 0x231), + (0x50224, 4), (0x5023c, 8), (0xa11d8, 0x740), (0xa1918, 0x231), + ], + text_sha256: [ + 0xc3, 0x53, 0x54, 0x40, 0x76, 0x02, 0x48, 0x44, 0x04, 0xaa, 0x6f, 0xc9, 0x46, + 0xcd, 0xd2, 0x4c, 0xb8, 0x2c, 0x64, 0xa8, 0xbb, 0xd1, 0x07, 0xa4, 0x94, 0x79, + 0xa0, 0x8c, 0xe9, 0xcc, 0x82, 0x51, + ], + }], + variant: &crate::g17::fw::variant::V26A428_M5, + config: None, + }]), + // Eight clusters of ten cores. M5 renders lay out their tiler context store themselves + // (`fw::m5::render_registers`), so the deflake layout is the M5 Pro's. As on the M5 Pro, the + // buffers are sized for all 80 core slots and the board admits the GPU by clusters and slots: + // parts with fused-off cores (64 enabled) take the same layout. + core_layout: CoreLayout { + cores: None, + alternate_core_masks: None, + operand_state: crate::g17::fw::compute::OperandState::EIGHT_BY_TEN, + deflake: crate::g17::fw::render::Deflake::TWO_CLUSTERS, + compute_blocks: crate::g17::fw::freelist::COMPUTE_BLOCKS, + }, +}; + +// Each compute operand bank holds 6 KiB per core slot of the board. +const _: () = { + use crate::g17::fw::m5::initdata::{J714, J716C, J775D}; + assert!(CONFIG.core_layout.operand_state.bank_bytes() == J714.gpu_cores * 0x1800); + assert!(CONFIG_MAX.core_layout.operand_state.bank_bytes() == J716C.gpu_cores * 0x1800); + assert!(CONFIG_ULTRA.core_layout.operand_state.bank_bytes() == J775D.gpu_cores * 0x1800); +}; + +/// Every T6050 board binds the same compatible; the ID registers select the configuration. +pub(crate) fn config(id: &G17Id) -> Option<&'static Config> { + [&CONFIG, &CONFIG_MAX, &CONFIG_ULTRA].into_iter().find(|cfg| id.family == cfg.identity.family + && id.variant == cfg.identity.variant && id.num_dies == cfg.identity.num_dies) +} diff --git a/drivers/gpu/drm/asahi/hw/t8140.rs b/drivers/gpu/drm/asahi/hw/t8140.rs index 72c172e14b818..891bb757db9e4 100644 --- a/drivers/gpu/drm/asahi/hw/t8140.rs +++ b/drivers/gpu/drm/asahi/hw/t8140.rs @@ -39,19 +39,21 @@ pub(crate) const CONFIG: Config = Config { ]), // Read by the firmware through the kernel lower root, whose top-level entries it only // samples when it starts. - pm_metrics: FixedBuffer { + profile: crate::g17::FirmwareProfile::PublishedRoots, + pm_metrics: Some(FixedBuffer { va: 0x10_0008_0000, size: 0x8000, - }, + }), recovery_slot_keys: Some(recovery::SLOT_KEYS), gpu_generation: GpuGen::G17, windows: ®ISTER_WINDOWS, reserved_slots: &RESERVED_REGISTER_SLOTS, - perf: &PERF, + perf: Some(&PERF), opps: OPPS, uat_quirks: crate::mmu::UatQuirks::DEFAULT, - region_c: &crate::g17::fw::initdata::REGION_C_26A428, + region_c: Some(&crate::g17::fw::initdata::REGION_C_26A428), chip: None, + m5: None, adopt_root_entries: true, fabric_bridge: true, firmware: crate::g17::fw::variant::Firmware::Fixed(&crate::g17::fw::variant::V26A428), @@ -329,7 +331,7 @@ pub(crate) mod resources { ]; pub(crate) const TVB_GROWTH: u64 = USER_BASE + 0x0400_0000; pub(crate) const TVB_PAIR_STRIDE: u64 = 0x2800_0000; - pub(crate) const RESERVED_RANGES: usize = 48; + pub(crate) const RESERVED_RANGES: usize = 50; /// Compact allocations search these shared client arenas in this order. pub(crate) fn compact_ranges() -> [Range; 4] { @@ -376,6 +378,13 @@ pub(crate) mod resources { } add(AUXILIARY); add(metrics_va..metrics_va.checked_add(METRICS_SIZE)?); + // The dynamic arenas the kernel places a client's queue aliases in (kick and + // descriptor rings, render generation state, client-private compute and render + // backing) stay reserved from creation too, as the fixed windows above are: an alias + // created later is then never mapped over a user bind, and the bind check's view of the + // reserved ranges cannot change between parsing a bind and committing it. + add(super::dynamic::LOWER); + add(super::dynamic::CLIENT_LOWER); Some(ranges) } } @@ -432,10 +441,12 @@ pub(crate) mod scheduling { /// Physical workqueue capacity and logical-owner admission limits. pub(crate) mod queues { pub(crate) const RENDER_SLOTS: usize = 64; - // One physical pair per logical owner leaves room for every execution - // context in the shared 128-QID namespace. Commands queue at RENDER_DEPTH, - // as many as a pair has command records (the render memory's RECORDS). - pub(crate) const RENDER_PAIRS_PER_OWNER: usize = 1; + // The QID budget reserves one physical pair per logical owner, which + // leaves room for every execution context in the shared 128-QID + // namespace. A second pair comes only from surplus, on G17D + // (g17::pairs). Commands queue at RENDER_DEPTH per pair, as many as a pair + // has command records (the render memory's RECORDS). + pub(crate) const RENDER_PAIRS_MAX: usize = 2; pub(crate) const RENDER_DEPTH: u8 = 32; // The firmware stores an ioctl's compute completion stamps only once all of the ioctl's // kicks finished, so finished commands hold their place until the ioctl's longest one ends. diff --git a/drivers/gpu/drm/asahi/m5/live_boot.rs b/drivers/gpu/drm/asahi/m5/live_boot.rs index 89bf0b08c7124..8f3c9f54325eb 100644 --- a/drivers/gpu/drm/asahi/m5/live_boot.rs +++ b/drivers/gpu/drm/asahi/m5/live_boot.rs @@ -279,6 +279,9 @@ fn new_uat(dev: &AsahiDrmDevice) -> Result<(Arc, [mmu::KernelMapping; Ok((uat, views)) } +/// Set once the module is pinned for an M5 session, which is retained until reboot. +static M5_MODULE_RETAINED: AtomicBool = AtomicBool::new(false); + pub(crate) struct M5Runtime { session: Option>, application: bool, @@ -321,8 +324,10 @@ impl M5Runtime { // even when firmware rejects the graph. Failed probe unwinding must // never release UAT or transports beneath a running firmware CPU. // Keep callback code resident even if a failed stop retains owners. - // This explicit RAM-boot experiment is released by reboot only. - unsafe { kernel::bindings::__module_get((*kernel::module::this_module::()).as_ptr()); } + // This explicit RAM-boot experiment is released by reboot only. No + // CPU has started yet: once module removal has begun, refuse with + // nothing started and the session freed. + crate::retain::retain_module(&M5_MODULE_RETAINED)?; Ok(Self { session: Some(session), application: false, faulted: false, application_generation: 0, application_compute_generation: 0 }) } diff --git a/drivers/gpu/drm/asahi/memacct.rs b/drivers/gpu/drm/asahi/memacct.rs index 12fc90b23d8eb..a61be841bff11 100644 --- a/drivers/gpu/drm/asahi/memacct.rs +++ b/drivers/gpu/drm/asahi/memacct.rs @@ -279,7 +279,8 @@ unsafe impl Send for Charge {} unsafe impl Sync for Charge {} impl Charge { - /// An object that is not counted (a dma-buf import). + /// An object that is not counted: a dma-buf import, or a kernel object whose owner holds a + /// user charge for it. pub(crate) fn none() -> Self { Self { kind: Kind::None, @@ -351,6 +352,33 @@ impl Charge { } } + /// Moves a user object's charge to `to`, the client now using it: its committed bytes, + /// within `to`'s limit, and its held bytes. The device totals and the process attribution + /// stay as they are. A refusal fails with `ENOMEM` and leaves the charge where it was. + /// Other kinds are attributed to no client and are left alone. + pub(crate) fn transfer(&mut self, to: &Arc) -> Result { + let bytes = self.bytes; + let held = *self.held.get_mut(); + let Kind::User { commit, .. } = &mut self.kind else { + return Ok(()); + }; + if Arc::ptr_eq(&commit.client, to) { + return Ok(()); + } + let ledger = commit.ledger; + if !try_add_within(&to.committed, bytes, ledger.client_limit.load(Ordering::Relaxed), 0) { + ledger.refused.fetch_add(1, Ordering::Relaxed); + return Err(ENOMEM); + } + if held { + to.resident.fetch_add(bytes, Ordering::AcqRel); + sub(&commit.client.resident, bytes); + } + sub(&commit.client.committed, bytes); + commit.client = to.clone(); + Ok(()) + } + /// Records that the GPU now holds (or no longer holds) the object's backing. /// /// Idempotent: only a change of state is counted. @@ -490,6 +518,36 @@ mod tests { Ok(()) } + static MOVE: Ledger = Ledger::new(); + + #[test] + fn a_transfer_moves_the_charge_once() -> Result { + MOVE.set_limits(Limits { device: 100 * MIB, client: 60 * MIB }); + let a = Client::new()?; + let b = Client::new()?; + let mut charge = Charge::user(MOVE.admit(&a, 40 * MIB)?, None); + charge.set_held(true); + let other = MOVE.admit(&b, 30 * MIB)?; + // b is within 60 MiB only without the 40 MiB: refused, and nothing moved. + assert_eq!(charge.transfer(&b), Err(ENOMEM)); + assert_eq!((a.committed(), a.resident(), b.committed()), (40 * MIB, 40 * MIB, 30 * MIB)); + drop(other); + charge.transfer(&b)?; + assert_eq!((a.committed(), a.resident()), (0, 0)); + assert_eq!((b.committed(), b.resident()), (40 * MIB, 40 * MIB)); + // The device counts the bytes once, before and after the move. + assert_eq!(MOVE.user_committed.load(Ordering::Relaxed), 40 * MIB); + assert_eq!(MOVE.user_resident.load(Ordering::Relaxed), 40 * MIB); + assert_eq!(MOVE.kernel_resident.load(Ordering::Relaxed), 0); + // Moving to the current client changes nothing; dropping uncharges the new client. + charge.transfer(&b)?; + drop(charge); + assert_eq!((b.committed(), b.resident()), (0, 0)); + assert_eq!(MOVE.user_committed.load(Ordering::Relaxed), 0); + assert_eq!(MOVE.user_resident.load(Ordering::Relaxed), 0); + Ok(()) + } + #[test] fn imports_are_not_counted() { let charge = Charge::none(); diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index 9ed3f8bec85ea..88f58892e12c1 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -112,6 +112,8 @@ const DEBUG_CLASS: DebugFlags = DebugFlags::Mmu; mod bind; mod context; +mod layout; + mod lifetime; #[cfg(CONFIG_DRM_ASAHI_M5)] mod m5_context; @@ -120,10 +122,12 @@ mod ppl; mod m3; mod residency; mod shared; +mod sparse_tile; +mod table; #[cfg(CONFIG_DRM_ASAHI_G18)] mod t8152; pub(crate) use bind::PreparedUserBindBatch; -pub(crate) use lifetime::VmJobGuard; +pub(crate) use lifetime::{RenderPoolMappings, VmJobGuard}; #[cfg(CONFIG_DRM_ASAHI_G18)] pub(crate) use t8152::G18ExecutionContext; /// The module-1134 name of the job guard G18 holds while a job uses a VM. @@ -223,6 +227,9 @@ pub(crate) struct UatQuirks { pub(crate) handoff_lock: bool, /// Handoff "current slot" written at initialization. pub(crate) handoff_slot: u32, + /// Handoff byte after the flush records written at initialization. It is 1 on T6050, where + /// it also turns the legacy flush protocol off. + pub(crate) handoff_unk2: u8, /// Store GPU-only mappings of client and kernel-lower tables as AP 0 with the access in /// PXN (read) and UXN (write), the form the firmware itself uses, instead of AP 2. pub(crate) native_gpu_permissions: bool, @@ -236,6 +243,7 @@ impl UatQuirks { page_tlbi: false, handoff_lock: true, handoff_slot: 0, + handoff_unk2: 0, native_gpu_permissions: false, }; } @@ -394,6 +402,16 @@ impl Drop for MappingMutation { } } +/// Where client-private aliases are placed: [`layout::CLIENT_PRIVATE_RANGE`] on M5, +/// outside every Mesa window; the A18 Pro's arena otherwise. +pub(crate) fn client_private_range(m5: bool) -> Range { + if m5 { + layout::CLIENT_PRIVATE_RANGE + } else { + crate::hw::t8140::dynamic::CLIENT_LOWER + } +} + /// Inner data for a Vm instance. struct VmInner { dev: driver::AsahiDevRef, @@ -407,6 +425,17 @@ struct VmInner { uat_inner: Arc, binding: Arc>, id: u64, + /// Per-VM sparse tiles (DRM_ASAHI_BIND_SPARSE_TILE): [read-only zero, + /// writable sink], allocated on first use. + sparse_tiles: [Option>; 2], +} + +/// GPUVA filled from a kernel sparse tile instead of its GEM's first page. +pub(crate) const GPUVA_SPARSE_TILE: u32 = kernel::bindings::drm_gpuva_flags_DRM_GPUVA_USERBITS; + +/// Whether a GPUVA carries [`GPUVA_SPARSE_TILE`]. +fn is_sparse_tile(flags: GpuVaFlags) -> bool { + u32::from(flags) & GPUVA_SPARSE_TILE != 0 } /// Exec-locked access to the [`VmInner`] of a GPUVM. @@ -616,6 +645,11 @@ impl gpuvm::DriverGpuVm for VmInner { if ctx.dev.is_none() { return Err(ENODEV); } + // A live leaf in the range belongs to another mapping, such as a kernel alias in a + // driver window: refuse before writing anything, so the rollback below never clears + // that mapping's leaves. + self.page_table + .ensure_unmapped(map_addr..map_addr + map_length as u64)?; // T8152 (module 1134): GPUVM cannot see driver-owned KernelMappings, and G18 places // driver aliases in user VMs outside the VM_CREATE kernel range. Preflight the full // object range in the authoritative page table (EBUSY on any populated leaf) and @@ -624,6 +658,19 @@ impl gpuvm::DriverGpuVm for VmInner { self.page_table .prepare_map(map_addr..map_addr.checked_add(map_length as u64).ok_or(EOVERFLOW)?)?; } + if is_sparse_tile(op.flags()) { + // Kernel tile fill: the GEM only anchors the GPUVA. + let result = if one_page { + self.map_sparse_tile(iova..iova + left as u64, ctx.prot) + } else { + Err(EINVAL) + }; + if let Err(error) = result { + self.rollback_failed_map(map_addr, map_length as u64); + return Err(error); + } + left = 0; + } // Bind preparation retained the table's backing before taking the VM reservation. // Never acquire the object reservation from inside the GPUVM map callback. let segments = op.vm_bo().data().segments.lock(); @@ -805,6 +852,23 @@ impl gpuvm::DriverGpuVm for VmInner { } impl VmInner { + /// Fill `range` from the kernel sparse tile for `prot`: the global zero + /// tile for read-only fills, this VM's own sink for writable ones (never + /// shared, so discarded writes cannot be read by another VM). The caller + /// holds the mapping mutation guard. + fn map_sparse_tile(&mut self, range: Range, prot: Prot) -> Result { + let kind = usize::from(prot != PROT_GPU_SHARED_RO); + let tile = match &self.sparse_tiles[kind] { + Some(tile) => tile.clone(), + None => { + let tile = sparse_tile::SparseTile::new()?; + self.sparse_tiles[kind] = Some(tile.clone()); + tile + } + }; + self.page_table.map_tiled(range, &tile, prot) + } + /// Remove hardware leaves for one complete user GPUVA without allocating. fn unmap_gpuva(&mut self, va: &gpuvm::GpuVa) -> Result { mod_dev_dbg!( @@ -956,6 +1020,21 @@ impl VmInner { self.tlbi_context_mask(iova, size, contexts); } + /// Invalidate a VA range in every TTBAT context that names this Vm's root in either half: a + /// kernel upper mapping reaches the firmware contexts and every execution context, a kernel + /// lower one the contexts that still name the kernel lower root. + fn tlbi_contexts_naming_root_in_either_half(&self, iova: u64, size: usize) { + if self.uat_inner.firmware == UatFirmware::Handoff { + return; + } + + let contexts = self + .uat_inner + .lock() + .contexts_naming_root_in_either_half(self.ttb()); + self.tlbi_context_mask(iova, size, contexts); + } + fn tlbi_context_mask(&self, iova: u64, size: usize, contexts: u64) { // Complete new leaves even if no context currently names this root: // a later context can publish it without repeating the map operation. @@ -990,6 +1069,12 @@ impl VmInner { self.page_table.prepare_map(node.start()..end)?; let uat_inner = self.uat_inner.clone(); let handoff = uat_inner.lock_m4_upper(self.is_kernel)?; + // Refuse a range holding another mapping's leaves before writing anything: the + // discard below clears the whole range. + if let Err(error) = self.page_table.ensure_unmapped(node.start()..end) { + drop(handoff); + return Err(error); + } let result = self.map_node_prepared(node, prot); let discard = if result.is_err() { // Rollback still changes live leaves and must remain under the PPL lease. @@ -1101,12 +1186,17 @@ pub(crate) struct Vm { #[cfg_attr(not(any(CONFIG_DRM_ASAHI_M5, CONFIG_DRM_ASAHI_G18)), allow(dead_code))] kernel_range: Range, mapping_epoch: Option>, + /// Created by an M5 GPU's UAT ([`Uat::is_m5`]). + m5: bool, bind_sequence: Arc, lifetime: Option>, residency: Option>, context_bindings: Option>, shared_bindings: Option>, status: Option>, + /// The memory accounting of the client that created this VM, which kernel memory sized + /// by that client's requests is charged to. `None` for kernel VMs. + memory: Option>, gpuvm: ARef>, dummy_obj: ARef, binding: Arc>, @@ -1123,6 +1213,14 @@ pub(crate) struct ExecutionContext { vm: Vm, } +/// One published command's claim on its context's hardware identity, held +/// until the command retires exactly. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct ContextPin { + pub(crate) id: u8, + pub(crate) generation: u64, +} + struct ExecutionRoot { released: AtomicBool, inner: Arc, @@ -1139,6 +1237,17 @@ impl ExecutionContext { pub(crate) fn generation(&self) -> u8 { self.root.generation } + /// Claim the context's hardware identity for one published command until + /// it retires exactly. + pub(crate) fn pin(&self) -> Result { + Ok(ContextPin { + id: self.root.id, + generation: self.root.generation.into(), + }) + } + + pub(crate) fn unpin(&self, _pin: ContextPin) {} + pub(crate) fn vm(&self) -> &Vm { &self.vm } @@ -1471,6 +1580,23 @@ impl KernelMapping { self.0.start()..(self.0.start() + self.0.mapped_size as u64) } + /// Whether the UAT tracks which TTBAT contexts name each root (published roots), so that + /// [`Self::publish_to_contexts`] reaches every context that can walk this mapping. Under the + /// handoff the caller keeps its own invalidation. + pub(crate) fn tracks_contexts(&self) -> bool { + self.0.uat_inner.firmware != UatFirmware::Handoff + } + + /// Publishes this mapping's page-table updates to every TTBAT context whose lower or upper + /// half names the owner's root. Call once the mapping set is complete, before work naming it + /// becomes visible. + pub(crate) fn publish_to_contexts(&self) { + self.0 + .owner + .lock_inner() + .tlbi_contexts_naming_root_in_either_half(self.iova(), self.size()); + } + /// Resolves a byte in this retained mapping without creating page tables. pub(crate) fn translate_offset(&self, offset: usize) -> Result { if offset >= self.size() { @@ -1553,9 +1679,10 @@ impl KernelMapping { } let _mutation = owner.mapping_mutation(); mutated = true; + // The prefix keeps the mapping's own access and changes only its cache attribute. owner.page_table.reprot_pages( self.iova()..self.iova() + prefix as u64, - PROT_GPU_FW_PRIV_RW, + self.0.prot.as_cached(), )?; mem::sync(); mem::tlbi_all(); @@ -1949,6 +2076,19 @@ impl UatShared { }) .fold(0, |mask, (ctx, _)| mask | (1 << ctx)) } + + /// Returns the mask of TTBAT contexts naming the root `ttb` in their lower or upper half. + fn contexts_naming_root_in_either_half(&self, ttb: u64) -> u64 { + self.ttbs() + .iter() + .enumerate() + .filter(|(ctx, slot)| { + let expected = tagged_root(ttb, *ctx); + slot.ttb0.load(Ordering::Acquire) == expected + || slot.ttb1.load(Ordering::Acquire) == expected + }) + .fold(0, |mask, (ctx, _)| mask | (1 << ctx)) + } } /// Returns the TTBAT value naming root `ttb` in context `ctx`. @@ -2111,6 +2251,8 @@ pub(crate) struct Uat { kernel_vm: Vm, kernel_lower_vm: Vm, + /// The GPU behind this UAT runs the M5 firmware's protocol ([`Uat::is_m5`]). + m5: bool, } impl Handoff { @@ -2149,7 +2291,7 @@ impl Handoff { } /// Initialize the handoff region - fn init(&self, firmware: UatFirmware, slot: u32) -> Result { + fn init(&self, firmware: UatFirmware, slot: u32, unk2: u8) -> Result { #[cfg(CONFIG_DRM_ASAHI_G18)] if firmware.is_no_peer() && (self.magic_fw.load(Ordering::Acquire) != 0 @@ -2160,6 +2302,7 @@ impl Handoff { } self.magic_ap.store(PPL_MAGIC, Ordering::Relaxed); self.cur_slot.store(slot, Ordering::Relaxed); + self.unk2.store(unk2, Ordering::Relaxed); self.unk3.store(0, Ordering::Relaxed); fence(Ordering::SeqCst); @@ -2358,6 +2501,7 @@ impl Vm { uat_inner, binding: binding.clone(), id, + sparse_tiles: [None, None], }, )?; let gpuvm = ARef::from(&*unique_gpuvm); @@ -2370,12 +2514,14 @@ impl Vm { kernel_range, gpuvm, mapping_epoch, + m5: false, bind_sequence, lifetime: None, residency: None, context_bindings: None, shared_bindings: None, status: None, + memory: None, dummy_obj: dummy_obj.gem.clone(), binding, }) @@ -2470,6 +2616,17 @@ impl Vm { } /// Status shared by every queue using this VM, if enabled by its GPU. + /// Charges kernel memory sized by this VM's client to `client` ([`Self::memory_client`]). + pub(crate) fn with_memory_client(mut self, client: Arc) -> Self { + self.memory = Some(client); + self + } + + /// The memory accounting of the client that created this VM, if it is a client VM. + pub(crate) fn memory_client(&self) -> Option<&Arc> { + self.memory.as_ref() + } + pub(crate) fn status(&self) -> Option<&Arc> { self.status.as_ref() } @@ -2971,6 +3128,13 @@ impl Vm { let transaction_uat = inner.uat_inner.clone(); let handoff = transaction_uat.lock_m4_upper(inner.is_kernel)?; let mutation = inner.mapping_mutation(); + // As for kernel mappings: refuse another mapping's leaves before the discard below + // could clear them. + if let Err(error) = inner.page_table.ensure_unmapped(iova..end) { + drop(mutation); + drop(handoff); + return Err(error); + } let ret = inner .page_table .map_pages(iova..end, phys as PhysicalAddr, prot, false); @@ -3433,6 +3597,7 @@ impl Uat { geometry, kernel_vm, kernel_lower_vm, + m5: false, inner, slots: slotalloc::SlotAllocator::new( UAT_USER_CTX as u32, @@ -3508,6 +3673,16 @@ impl Uat { &self.kernel_lower_vm } + /// Links the kernel lower root's tables for each range, with no mappings. A published-roots + /// firmware samples that root's top-level entries only when it starts. + pub(crate) fn prepare_lower_tables(&self, ranges: &[Range]) -> Result { + let mut inner = lock_vm(&self.kernel_lower_vm.gpuvm, None, false)?; + for range in ranges { + inner.page_table.alloc_pages(range.clone())?; + } + Ok(()) + } + /// Installed descriptor and bind contexts rooted in this user VM. A graph /// constructor checks this before changing a retained pool's mappings. pub(crate) fn vm_context_mask(&self, vm: &Vm) -> Result { @@ -3715,8 +3890,16 @@ impl Uat { None => return Err(ENOSPC), }; inner.execution_contexts |= 1u64 << id; - inner.context_generations[id] = inner.context_generations[id].wrapping_add(1); - (id, inner.context_generations[id]) + // On M5 a slot's reuse generation is never zero, which M5's context token reserves + // for none: it wraps from 255 to 1. Slots reused past 255 times (reclaimed M5 queues) + // would otherwise hand out zero. + let generation = &mut inner.context_generations[id]; + *generation = if self.m5 { + generation.checked_add(1).unwrap_or(1) + } else { + generation.wrapping_add(1) + }; + (id, *generation) }; let low = tagged_root(vm.ttb(), id); let high = tagged_root(self.kernel_vm.ttb(), id); @@ -3755,7 +3938,7 @@ impl Uat { /// Creates a new `Vm` linked to this UAT. pub(crate) fn new_vm(&self, id: u64, kernel_range: Range) -> Result { self.inner.check_m4()?; - let vm = Vm::new( + let mut vm = Vm::new( self.dev.clone(), self.inner.clone(), kernel_range, @@ -3763,10 +3946,51 @@ impl Uat { None, id, )?; + vm.m5 = self.m5; self.inner.check_m4()?; Ok(vm) } + /// The GPU behind this UAT runs the M5 firmware's protocol. + pub(crate) fn is_m5(&self) -> bool { + self.m5 + } + + /// Marks a published-roots UAT as an M5 GPU's ([`Uat::is_m5`]). + pub(crate) fn with_m5(mut self) -> Self { + self.m5 = true; + self + } + + /// Keep dynamic objects outside the M5 boot graph, its unmapped sentinel, + /// and the firmware register aperture. Client mappings have their own VA. + pub(crate) fn kernel_dynamic_range(&self) -> Range { + let mut range = self.geometry.kernel_range(); + if self.is_m5() { + range.start += 0x2_0000_0000; + range + } else { + crate::hw::t8140::dynamic::upper(range) + } + } + + /// Canonical lower objects are aliased into every owning client VM. + /// Sparse VA reservation has no physical-memory cost. + pub(crate) fn kernel_lower_dynamic_range(&self) -> Range { + if self.is_m5() { + layout::CANONICAL_RANGE + } else { + crate::hw::t8140::dynamic::LOWER + } + } + + /// The MMU's arenas an M5 client VM keeps from userspace binds, even while they + /// hold nothing: the canonical arena its queues alias and the client-private one. + /// The client VM also reserves the compact window (`BlockGeometry::M5_COMPACT`). + pub(crate) fn m5_client_reserved_ranges(&self) -> [Range; 2] { + [layout::CANONICAL_RANGE, layout::CLIENT_PRIVATE_RANGE] + } + /// Returns the address space geometry of this UAT. pub(crate) fn geometry(&self) -> UatGeometry { self.geometry @@ -4036,6 +4260,7 @@ impl Uat { geometry, kernel_vm, kernel_lower_vm, + m5: false, inner, slots: slotalloc::SlotAllocator::new( UAT_USER_CTX as u32, @@ -4053,7 +4278,7 @@ impl Uat { inner.kernel_ttb1 = ttb1; inner.secondary_ttb1 = secondary_ttb1; - inner.handoff().init(firmware, quirks.handoff_slot)?; + inner.handoff().init(firmware, quirks.handoff_slot, quirks.handoff_unk2)?; if quirks.clean_tables { pgtable::clean_poc( core::ptr::from_ref(inner.handoff()).cast::(), diff --git a/drivers/gpu/drm/asahi/mmu/bind.rs b/drivers/gpu/drm/asahi/mmu/bind.rs index acce5677b9f92..a792cdd1bec08 100644 --- a/drivers/gpu/drm/asahi/mmu/bind.rs +++ b/drivers/gpu/drm/asahi/mmu/bind.rs @@ -15,6 +15,7 @@ pub(crate) struct PreparedUserMap { size: u64, offset: u64, single_page: bool, + sparse_tile: bool, context: context::PreparedBinding, _residency: Option, _sg_lease: kernel::drm::gem::shmem::SGLease, @@ -59,6 +60,9 @@ impl PreparedUserBindBatch { impl Vm { /// Prepares host state before taking the GPUVM exec lock or changing any leaf. + /// With `sparse_tile` (DRM_ASAHI_BIND_SPARSE_TILE) the range is filled from a kernel + /// sparse tile and `gem` only anchors it; it requires `single_page`. + #[allow(clippy::too_many_arguments)] pub(crate) fn prepare_bind_object( &self, gem: &gem::Object, @@ -67,7 +71,11 @@ impl Vm { offset: u64, prot: Prot, single_page: bool, + sparse_tile: bool, ) -> Result { + if sparse_tile && !single_page { + return Err(EINVAL); + } gem::validate_vm_binding(gem, self)?; let residency = self.enter_residency_metadata()?; let context = self.prepare_context_binding(addr, size, offset)?; @@ -103,6 +111,7 @@ impl Vm { size, offset, single_page, + sparse_tile, context, }) } @@ -138,7 +147,12 @@ impl Vm { let mut inner = unsafe { self.gpuvm.exec_lock_vm(None, true)? }; match self.mapping_commit() { Ok(commit) => { - let (flags, repeat) = if map.single_page { + let (flags, repeat) = if map.sparse_tile { + ( + GpuVaFlags::from(GpuVaFlag::REPEAT) | GpuVaFlags::from(super::GPUVA_SPARSE_TILE), + UAT_PGSZ as u32, + ) + } else if map.single_page { (GpuVaFlag::REPEAT.into(), UAT_PGSZ as u32) } else { (GpuVaFlags::empty(), 0) diff --git a/drivers/gpu/drm/asahi/mmu/layout.rs b/drivers/gpu/drm/asahi/mmu/layout.rs new file mode 100644 index 0000000000000..6bd1a17df97a8 --- /dev/null +++ b/drivers/gpu/drm/asahi/mmu/layout.rs @@ -0,0 +1,14 @@ +// SPDX-License-Identifier: GPL-2.0-only + +//! Fixed GPU VA arenas of the M5 lower address space. + +use core::ops::Range; + +/// Canonical queue backing has its own sparse arena. It must not overlap +/// the client-private arena or retain the A18 Pro's sub-2-GiB limit. +pub(super) const CANONICAL_RANGE: Range = 0x80_0000_0000..0x90_0000_0000; +/// Client-private aliases of M5 queues, placed per client VM. Like the +/// canonical arena it lies above Mesa's M5 windows (driver reservation, data +/// heap 0x40.., its read-only shadow 0x60..0x80, which Mesa binds for every +/// allocation, and the USC window 0x100..), so no user mapping can need it. +pub(super) const CLIENT_PRIVATE_RANGE: Range = 0x90_0000_0000..0xa0_0000_0000; diff --git a/drivers/gpu/drm/asahi/mmu/lifetime.rs b/drivers/gpu/drm/asahi/mmu/lifetime.rs index 66d507c2d7694..9da7820dd92f6 100644 --- a/drivers/gpu/drm/asahi/mmu/lifetime.rs +++ b/drivers/gpu/drm/asahi/mmu/lifetime.rs @@ -28,6 +28,10 @@ struct PoolMappings { mappings: KVec, } +/// A graph's pool mappings moved out of a VM slot (see +/// Vm::take_render_pool_mappings). Dropping it unmaps them. +pub(crate) struct RenderPoolMappings(PoolMappings); + impl DriverMappings { fn ranges(&self) -> impl Iterator> + '_ { Iterator::chain( @@ -170,6 +174,51 @@ impl Vm { Ok(()) } + /// Move a drained graph's pool mappings for `slot` out of this VM + /// (so another graph of the same VM can install there); `put` returns them + /// if that install fails. Only the exact owner identity is taken. + pub(crate) fn take_render_pool_mappings(&self, slot: u8, id: u64) -> Option { + if id == 0 { + return None; + } + let lifetime = self.lifetime.as_ref()?; + let mut driver = lifetime.driver.lock(); + Option::as_mut(&mut *driver)? + .render + .get_mut(usize::from(slot))? + .take_if(|pool| pool.id == id) + .map(RenderPoolMappings) + } + pub(crate) fn put_render_pool_mappings(&self, slot: u8, mappings: RenderPoolMappings) -> Result { + let lifetime = self.lifetime.as_ref().ok_or(EINVAL)?; + let mut driver = lifetime.driver.lock(); + let driver = Option::as_mut(&mut *driver).ok_or(EINVAL)?; + let pool = driver.render.get_mut(usize::from(slot)).ok_or(EINVAL)?; + if pool.is_some() { + return Err(EBUSY); + } + *pool = Some(mappings.0); + Ok(()) + } + + /// The graph identity whose pool mappings occupy `slot` in this + /// VM (0 when free). A second graph cannot install there until it leaves. + pub(crate) fn render_pool_owner(&self, slot: u8) -> u64 { + self.lifetime + .as_ref() + .and_then(|lifetime| { + lifetime + .driver + .lock() + .as_ref()? + .render + .get(usize::from(slot))? + .as_ref() + .map(|pool| pool.id) + }) + .unwrap_or(0) + } + pub(crate) fn render_pool_tvb_blocks(&self, slot: u8, id: u64) -> Option { if id == 0 { return None; @@ -652,6 +701,31 @@ impl Drop for VmJobGuard { if state.active != 0 { return; } + // Healthy last-job release need not close mapping/job admission when + // there is no accepted cleanup to drain. Keep every recovery, close, + // in-progress commit and failed-cleanup path on the existing drain. + // M5 only: other G17 GPUs keep the drain on every last release. + if self.vm.m5 + && !state.blocked + && !state.quarantined + && !state.draining + && !state.closed + && !state.cleanup_failed + && state.commits == 0 + && state.close_ranges.is_none() + && state.unmaps.is_empty() + && state.objects.is_empty() + && self.vm.status.as_ref().is_some_and(|status| { + status.get() == 0 && status.reported_error() == 0 + }) + { + drop(state); + // The C deferred list is independent of the queues above. Its + // GEM puts may sleep/reenter; run after unlocking, while this + // job guard still retains the VM and its residency lease. + self.vm.bo_deferred_cleanup(); + return; + } state.draining = true; drop(state); self.vm.drain_mappings(lifetime); diff --git a/drivers/gpu/drm/asahi/mmu/residency.rs b/drivers/gpu/drm/asahi/mmu/residency.rs index 1df852184b31d..a83068a5c9e9f 100644 --- a/drivers/gpu/drm/asahi/mmu/residency.rs +++ b/drivers/gpu/drm/asahi/mmu/residency.rs @@ -616,6 +616,14 @@ impl Vm { inner.page_table.discard_partial_map(map.addr..end)?; inner.tlbi_contexts_naming_root(map.addr, map.range as usize); let prot = map.inner.prot.ok_or(EINVAL)?; + if super::is_sparse_tile(map.flags) { + // A tile fill has no GEM pages to restore: reinstall it. + let _change = inner.mapping_mutation(); + inner.map_sparse_tile(map.addr..end, prot)?; + inner.tlbi_contexts_naming_root(map.addr, map.range as usize); + inner.set_invalidated(map.addr, map.range, false)?; + return Ok(()); + } let repeat = map.flags.contains(GpuVaFlag::REPEAT); let mut skip = map .offset diff --git a/drivers/gpu/drm/asahi/mmu/shared.rs b/drivers/gpu/drm/asahi/mmu/shared.rs index fec7a6b61ad8f..a5ea381434d44 100644 --- a/drivers/gpu/drm/asahi/mmu/shared.rs +++ b/drivers/gpu/drm/asahi/mmu/shared.rs @@ -108,11 +108,12 @@ impl Vm { pub(crate) fn render_footprint( &self, attachments: &Attachments, + whole: bool, ) -> Result>> { if attachments.as_slice().is_empty() { return Ok(None); } - let mut footprint = Footprint::new(self.id); + let mut footprint = Footprint::new(self.id, whole); let bindings = self.shared_bindings.as_ref().and_then(|shared| { (shared.count.load(Ordering::Relaxed) != 0).then(|| shared.bindings.lock()) }); diff --git a/drivers/gpu/drm/asahi/mmu/sparse_tile.rs b/drivers/gpu/drm/asahi/mmu/sparse_tile.rs new file mode 100644 index 0000000000000..dc4a5459eff10 --- /dev/null +++ b/drivers/gpu/drm/asahi/mmu/sparse_tile.rs @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: GPL-2.0-only + +//! Kernel-owned scratch for sparse fills (DRM_ASAHI_BIND_SPARSE_TILE). + +use kernel::{prelude::*, sync::Arc}; + +/// A zeroed, physically contiguous kernel allocation. The page tables repeat +/// its first page across a filled range, as a SINGLE_PAGE bind repeats its +/// GEM's first page. +pub(super) struct SparseTile { + page: *mut kernel::bindings::page, + physical: u64, +} + +// SAFETY: The pages are exclusively owned until Drop; only their physical +// address is shared, and it is immutable. +unsafe impl Send for SparseTile {} +// SAFETY: See Send. No interior mutability. +unsafe impl Sync for SparseTile {} + +impl SparseTile { + /// One UAT page, the only part the page tables read: the order of the kernel pages that + /// make up a UAT page, naturally aligned to it. + const ORDER: u32 = (crate::pgtable::UAT_PGSZ / kernel::page::PAGE_SIZE).trailing_zeros(); + + pub(super) fn new() -> Result> { + // SAFETY: A plain zeroed higher-order page allocation. + let page = unsafe { kernel::bindings::alloc_pages( + kernel::bindings::GFP_KERNEL | kernel::bindings::__GFP_ZERO, Self::ORDER) }; + if page.is_null() { return Err(ENOMEM); } + // SAFETY: page is a live allocation returned above. + let physical = unsafe { kernel::bindings::page_to_phys(page) } as u64; + let tile = Arc::new(Self { page, physical }, GFP_KERNEL); + if tile.is_err() { + // SAFETY: The allocation above was not published anywhere. + unsafe { kernel::bindings::__free_pages(page, Self::ORDER) }; + } + Ok(tile?) + } + + pub(super) fn physical(&self) -> u64 { self.physical } +} + +impl Drop for SparseTile { + fn drop(&mut self) { + // SAFETY: The VM that maps the tile owns it and drops it after its + // page tables. + unsafe { kernel::bindings::__free_pages(self.page, Self::ORDER) }; + } +} diff --git a/drivers/gpu/drm/asahi/mmu/table.rs b/drivers/gpu/drm/asahi/mmu/table.rs new file mode 100644 index 0000000000000..53d6c30273b5e --- /dev/null +++ b/drivers/gpu/drm/asahi/mmu/table.rs @@ -0,0 +1,24 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Page-table operations the M5 paths use beyond the common ones. + +use super::{sparse_tile::SparseTile, Prot, UatPageTable}; +use core::ops::Range; +use kernel::{addr::PhysicalAddr, prelude::*, sync::Arc}; + +impl UatPageTable { + pub(super) fn visit_runs( + &self, + range: Range, + visit: impl FnMut(u64, u64, u8) -> Result, + ) -> Result { + let _ = (range, visit); + Err(EOPNOTSUPP) + } + + /// Fill `range` from a sparse tile: its first (zeroed) page repeats across + /// the range, as a SINGLE_PAGE bind repeats its GEM's first page. + pub(super) fn map_tiled(&mut self, range: Range, tile: &Arc, prot: Prot) -> Result { + self.map_pages(range, tile.physical() as PhysicalAddr, prot, true) + } +} diff --git a/drivers/gpu/drm/asahi/pgtable.rs b/drivers/gpu/drm/asahi/pgtable.rs index 0f915607cb1f4..f1980e6c4f4f9 100644 --- a/drivers/gpu/drm/asahi/pgtable.rs +++ b/drivers/gpu/drm/asahi/pgtable.rs @@ -289,7 +289,7 @@ const HIGH_BITS_PXN: u16 = 1 << 1; const HIGH_BITS_UXN: u16 = 1 << 2; const HIGH_BITS_GPU_ACCESS: u16 = 1 << 3; -#[derive(Debug, Copy, Clone)] +#[derive(Debug, Copy, Clone, PartialEq, Eq)] pub(crate) struct Prot { memattr: u8, ap: u8, @@ -417,6 +417,10 @@ impl Prot { pub(crate) const fn as_uncached(&self) -> Self { self.memattr(MEMATTR_UNCACHED) } + + pub(crate) const fn as_cached(&self) -> Self { + self.memattr(MEMATTR_CACHED) + } } impl Default for Prot { @@ -669,6 +673,7 @@ impl UatPageTable { mod_pr_debug!("UATPageTable::with_pages: borrowed phys {:#x}\n", phys); let driver_tables = &mut self.driver_tables; let oas_mask = self.oas_mask; + let firmware_tree = reserved.as_deref(); let entry = |p: *mut u8| -> Result> { let uptep = p.cast_const().cast::(); // SAFETY: with_table() ensures the pointer is valid, and our index is @@ -706,7 +711,21 @@ impl UatPageTable { } } if upte_val & PTE_TYPE_BITS == PTE_TYPE_LEAF_TABLE { - Ok(Some(upte_val & oas_mask & (!UAT_PGMSK as u64))) + let child = upte_val & oas_mask & (!UAT_PGMSK as u64); + // Below a root the driver did not allocate, the descriptors were + // written by the firmware or the bootloader. Follow one only into a + // reserved table page or a table this driver allocated, never into + // other RAM, whose PTEs this walk would then write. + if let (Some(firmware_tree), Some(tables)) = + (firmware_tree, driver_tables.as_ref()) + { + if !firmware_tree.contains(child) && !tables.contains(&child) { + pr_err!("UATPageTable::with_pages: table PTE {:#x} at iova {:#x} level {} names a page outside the reserved and driver tables\n", + upte_val, iova, level + 1); + return Err(EIO); + } + } + Ok(Some(child)) } else if upte_val == 0 || (!alloc && !free) { mod_pr_debug!("UATPageTable::with_pages: no level {}\n", level); Ok(None) @@ -986,11 +1005,15 @@ impl UatPageTable { for (idx, pte) in ptes.iter().enumerate() { let ptev = pte.load(Ordering::Relaxed); if ptev != 0 { + // Never replace a live leaf: it belongs to another mapping. The callers + // check their whole range first (`ensure_unmapped`), so this is reached + // only by a collision they missed. pr_err!( "UATPageTable::map_pages: Page at IOVA {:#x} is mapped (PTE: {:#x})\n", iova + (idx * UAT_PGSZ) as u64, ptev ); + return Err(EBUSY); } pte.store(phys | prot.as_pte() | pte_bits, Ordering::Relaxed); if !one_page { @@ -1001,6 +1024,28 @@ impl UatPageTable { }) } + /// Refuses with `EBUSY`, writing nothing, when any leaf of `iova_range` is live. + /// + /// Every PTE store writes either zero or a complete entry, so a nonzero leaf belongs to + /// another mapping. A map operation checks its whole range with this before its first + /// store: its failure path clears the whole range it asked for, which would otherwise + /// also clear the other mapping's leaves. + pub(crate) fn ensure_unmapped(&mut self, iova_range: Range) -> Result { + self.walk(iova_range, false, false, false, |iova, ptes| { + match ptes.iter().position(|pte| pte.load(Ordering::Relaxed) != 0) { + Some(idx) => { + pr_err!( + "UATPageTable::ensure_unmapped: Page at IOVA {:#x} is mapped (PTE: {:#x})\n", + iova + (idx * UAT_PGSZ) as u64, + ptes[idx].load(Ordering::Relaxed) + ); + Err(EBUSY) + } + None => Ok(()), + } + }) + } + pub(crate) fn reprot_pages(&mut self, iova_range: Range, prot: Prot) -> Result { mod_pr_debug!( "UATPageTable::reprot_pages: {:#x?} {:?}\n", diff --git a/drivers/gpu/drm/asahi/queue_share.rs b/drivers/gpu/drm/asahi/queue_share.rs new file mode 100644 index 0000000000000..b7a8ff42affff --- /dev/null +++ b/drivers/gpu/drm/asahi/queue_share.rs @@ -0,0 +1,161 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Each process's share of the GPU's independent queues. +//! +//! A process can open the device as often as it likes, so a queue limit per open file limits +//! nothing. Queues are therefore counted, device-wide, against the process that created them, +//! identified by its mm as [`crate::memacct`] attributes memory, for as long as they exist. +//! +//! A process may always hold up to half of the device's queues. It may take more while at least +//! [`reserve`] queues stay free for the other processes (the compositor and anything started +//! later), or while no other process holds a queue, so a process alone can still use every one. +//! A process below half its share is never refused here, so the reserve stays open to them. + +use kernel::{ + mm::Mm, + prelude::*, + sync::aref::ARef, // +}; + +kernel::sync::global_lock! { + // SAFETY: Module initialization initializes this lock before registering adapters. + unsafe(uninit) static PROCESSES: Mutex> = KVec::new(); +} + +/// Initialize the registry before a platform adapter can probe. +/// +/// # Safety +/// +/// Called exactly once per module instance, before any queue is created. +pub(crate) unsafe fn init() { + // SAFETY: The module initializer establishes the single-call ordering above. + unsafe { PROCESSES.init() }; +} + +/// The queues one process holds. +pub(crate) struct Process { + key: usize, + /// Keeps the `mm_struct`, and with it `key`, from being reused while queues count against it. + _mm: Option>, + held: u32, +} + +/// Queues kept free for the other processes once one process holds half of the device's `max`. +fn reserve(max: u32) -> u32 { + (max / 8).max(1) +} + +/// Whether a process holding `held` of the `total` queues counted device-wide may take another +/// of the device's `max`. The device itself refuses a queue it has no room for. +fn may_take(held: u32, total: u32, max: u32) -> bool { + held < max / 2 || held == total || max.saturating_sub(total.saturating_add(1)) >= reserve(max) +} + +/// Counts a new queue against the process `key`; EBUSY when [`may_take`] refuses it. +fn take(processes: &mut KVec, key: usize, mm: Option>, max: u32) -> Result { + let total = processes.iter().map(|process| process.held).sum::(); + let index = processes.iter().position(|process| process.key == key); + let held = index.map_or(0, |index| processes[index].held); + if !may_take(held, total, max) { + return Err(EBUSY); + } + match index { + Some(index) => processes[index].held += 1, + None => processes.push(Process { key, _mm: mm, held: 1 }, GFP_KERNEL)?, + } + Ok(()) +} + +/// Gives back one queue of `key`; returns the process once it holds none, to drop unlocked. +fn give_back(processes: &mut KVec, key: usize) -> Option { + let index = processes.iter().position(|process| process.key == key)?; + processes[index].held -= 1; + (processes[index].held == 0).then(|| processes.swap_remove(index)) +} + +/// One queue counted against the process that created it; dropping it gives the queue back. +pub(crate) struct Share { + key: usize, +} + +impl Share { + /// Counts a new queue against the current process, of the device's `max` queues. + pub(crate) fn take_current(max: u32) -> Result { + let mm = kernel::current!().mm().map(|mm| ARef::from(&**mm)); + let key = mm.as_ref().map_or(0, |mm| mm.as_raw() as usize); + take(&mut PROCESSES.lock(), key, mm, max)?; + Ok(Self { key }) + } +} + +impl Drop for Share { + fn drop(&mut self) { + let gone = give_back(&mut PROCESSES.lock(), self.key); + // Drop the process, and with it the mm reference, after the lock. + drop(gone); + } +} + +#[kunit_tests(asahi_queue_share)] +mod tests { + use super::*; + + #[test] + fn a_process_alone_may_use_every_queue() { + assert!(may_take(57, 57, 58)); + assert!(may_take(41, 41, 42)); + } + + #[test] + fn a_large_holder_leaves_the_reserve_to_others() { + // 58 queues keep 7 free for the others; 42 keep 5. + assert_eq!(reserve(58), 7); + assert_eq!(reserve(42), 5); + // With a compositor holding 3, one process gets to 48 of 58, not 29. + assert!(may_take(47, 50, 58)); + assert!(!may_take(48, 51, 58)); + // Below half its share, a process may take queues from the reserve. + assert!(may_take(3, 51, 58)); + assert!(may_take(28, 57, 58)); + } + + #[test] + fn a_lone_process_with_two_files_reaches_the_full_budget() -> Result { + let mut processes = KVec::new(); + // Another process held queues and gave them all back: it no longer counts. + take(&mut processes, 2, None, 58)?; + assert!(give_back(&mut processes, 2).is_some()); + assert!(processes.is_empty()); + // Both files of process 1 count against one key, which may take all 58. + for _ in 0..58 { + take(&mut processes, 1, None, 58)?; + } + assert_eq!(processes.len(), 1); + assert_eq!(processes[0].held, 58); + Ok(()) + } + + #[test] + fn reopening_does_not_reset_the_count() -> Result { + let mut processes = KVec::new(); + // One process (key 1) beside a compositor (key 2) holding 3 queues of 58. + for _ in 0..3 { + take(&mut processes, 2, None, 58)?; + } + // Every file of process 1 counts against the same key. + for _ in 0..48 { + take(&mut processes, 1, None, 58)?; + } + assert_eq!(take(&mut processes, 1, None, 58), Err(EBUSY)); + // The compositor still gets one, and so does a new process. + take(&mut processes, 2, None, 58)?; + take(&mut processes, 3, None, 58)?; + // Giving every queue back removes the process. + for _ in 0..47 { + assert!(give_back(&mut processes, 1).is_none()); + } + assert!(give_back(&mut processes, 1).is_some()); + assert!(processes.iter().all(|process| process.key != 1)); + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/regs.rs b/drivers/gpu/drm/asahi/regs.rs index 976c789930068..60e485f29f06b 100644 --- a/drivers/gpu/drm/asahi/regs.rs +++ b/drivers/gpu/drm/asahi/regs.rs @@ -625,6 +625,55 @@ impl G17Resources { Ok(true) } + /// Publishes through the M5 host command port, including the kick that + /// makes an idle queue runnable. Core activity is not a prerequisite. + pub(crate) fn submit_m5_work( + &self, + submission: &crate::g17::fw::m5::queue::HardwareSubmission, + ) -> Result { + let sgx = self.sgx.try_access().ok_or(ENODEV)?; + let sgx = (&*sgx).as_view(); + for &(offset, value) in submission.writes() { + sgx.try_write64(value, offset)?; + } + Ok(()) + } + + /// The M5 fault window, read for recovery evidence: the fault status, the MMU fault status + /// and address, and three further fault words. The selector is observed, never changed. + pub(crate) fn m5_fault_registers(&self) -> Result<[u64; 6]> { + let sgx = self.sgx.try_access().ok_or(ENODEV)?; + let sgx = (&*sgx).as_view(); + if sgx.relaxed().read32(G17_HOST_IRQ_SUMMARY) & G17_HOST_POWERED == 0 { + return Err(EAGAIN); + } + Ok([ + sgx.try_read64(0xd800)?, + sgx.try_read64(0xd8c0)?, + sgx.try_read64(0xd8c8)?, + sgx.try_read64(0xc078)?, + sgx.try_read64(0xc008)?, + sgx.try_read64(0xc028)?, + ]) + } + + /// Sample only the two owned queue configuration words through the + /// host tag/data port. The caller holds the device mutex and power hold. + pub(crate) fn m5_queue_configurations(&self, qids: [u8; 2]) -> Result<[u64; 2]> { + let sgx = self.sgx.try_access().ok_or(ENODEV)?; + let sgx = (&*sgx).as_view(); + if sgx.relaxed().read32(G17_HOST_IRQ_SUMMARY) & G17_HOST_POWERED == 0 { + return Err(EAGAIN); + } + let mut values = [0; 2]; + for (value, qid) in values.iter_mut().zip(qids) { + let tag = crate::g17::fw::m5::queue::HardwareSubmission::configuration_tag(qid)?; + sgx.try_write64(tag, 0x4090)?; + *value = sgx.try_read64(0x4098)?; + } + Ok(values) + } + /// Registers a queue KSM slot through the firmware mailbox. pub(crate) fn register_ksm_queue(&self, qid: u8, descriptor: u64) -> Result { if qid > 127 { @@ -821,6 +870,22 @@ impl G17Resources { /// Enable dynamic power gating of idle G17 GPU cores. The firmware expects this before it is /// initialized. + /// Prepares the GPU for the M5 firmware before its initialization: sets bit 0 of the + /// firmware start control, which reads 0x30000 or 0x30001 until then, and enables dynamic + /// core power gating. + pub(crate) fn prepare_m5_firmware(&self) -> Result { + { + let sgx = self.sgx.try_access().ok_or(ENODEV)?; + let sgx = (&*sgx).as_view(); + let value = sgx.try_read32(0xe90000)?; + if value != 0x30000 && value != 0x30001 { + return Err(EIO); + } + sgx.try_write32(value | 1, 0xe90000)?; + } + self.enable_g17_gating() + } + pub(crate) fn enable_g17_gating(&self) -> Result { let sgx = self.sgx.try_access().ok_or(ENODEV)?; let sgx = (&*sgx).as_view(); diff --git a/drivers/gpu/drm/asahi/retain.c b/drivers/gpu/drm/asahi/retain.c new file mode 100644 index 0000000000000..66b1548a7bc3c --- /dev/null +++ b/drivers/gpu/drm/asahi/retain.c @@ -0,0 +1,26 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +// +// THIS_MODULE for the once-per-boot module pin (retain.rs): a firmware +// session that is never freed keeps callbacks into this module. + +#include +#include + +bool asahi_module_try_get(void); +void asahi_module_put(void); + +/* + * Pin this module until reboot, unless its removal has begun: a deferred or + * asynchronous probe can run after an ordinary rmmod has passed its reference + * check, which __module_get() would not stop. + */ +bool asahi_module_try_get(void) +{ + return try_module_get(THIS_MODULE); +} + +/* Drop a reference asahi_module_try_get() took. */ +void asahi_module_put(void) +{ + module_put(THIS_MODULE); +} diff --git a/drivers/gpu/drm/asahi/g15/retain.rs b/drivers/gpu/drm/asahi/retain.rs similarity index 77% rename from drivers/gpu/drm/asahi/g15/retain.rs rename to drivers/gpu/drm/asahi/retain.rs index 30fd79d510869..9df6f417be2fe 100644 --- a/drivers/gpu/drm/asahi/g15/retain.rs +++ b/drivers/gpu/drm/asahi/retain.rs @@ -2,15 +2,23 @@ //! The once-per-boot module pin taken before the firmware is first touched. //! -//! A started G15 firmware session cannot be stopped, so this module must stay loaded until -//! reboot from the first firmware-visible write on. `__module_get` alone does not ensure that: -//! a deferred or asynchronous probe can run while an ordinary rmmod is already past its -//! reference check, and the module is then unloaded under a running firmware. -//! `try_module_get` fails once removal has begun, so the pin is taken with it, before the -//! session is marked retained and before anything firmware-visible happens. +//! A started firmware session that is never freed (every backend's: G15, G16, G18, the M5 +//! RAM-boot path, and G17 on M5) keeps RTKit and mailbox callbacks into this module, so the +//! module must stay loaded until reboot from the first firmware-visible write on. +//! `__module_get` alone does not ensure that: a deferred or asynchronous probe can run while +//! an ordinary rmmod is already past its reference check, and the module is then unloaded +//! under a running firmware. `try_module_get` fails once removal has begun, so the pin is +//! taken with it, before the session is marked retained and before anything +//! firmware-visible happens. use core::sync::atomic::{AtomicBool, Ordering}; +#[cfg(not(test))] +extern "C" { + fn asahi_module_try_get() -> bool; + fn asahi_module_put(); +} + /// The module is being removed: nothing was pinned or marked, and nothing may be started. #[derive(Debug, PartialEq, Eq)] pub(crate) struct Removing; @@ -38,6 +46,21 @@ pub(crate) fn retain( Ok(()) } +/// Pins this module until reboot, once for the session `retained` tracks +/// (`try_module_get(THIS_MODULE)`, see [`retain`]). ENODEV, with nothing pinned or marked, once +/// the module's removal has begun. +#[cfg(not(test))] +pub(crate) fn retain_module(retained: &AtomicBool) -> kernel::error::Result { + retain( + retained, + // SAFETY: Takes a reference to this module unless its removal has begun. + || unsafe { asahi_module_try_get() }, + // SAFETY: Drops only the reference this call took; the winner's is never put. + || unsafe { asahi_module_put() }, + ) + .map_err(|_| kernel::error::code::ENODEV) +} + #[cfg(test)] mod tests { use super::*; diff --git a/drivers/hwmon/macsmc-hwmon.c b/drivers/hwmon/macsmc-hwmon.c index ddf287a190efa..0fd601dfa4162 100644 --- a/drivers/hwmon/macsmc-hwmon.c +++ b/drivers/hwmon/macsmc-hwmon.c @@ -90,6 +90,8 @@ struct macsmc_hwmon_fan { bool manual; /* The device tree allows a manual speed from userspace. */ bool manual_control; + /* Manual for a kernel thermal policy (macsmc_hwmon_fans_request()). */ + bool kernel; }; struct macsmc_hwmon_sensors { @@ -457,6 +459,7 @@ static int macsmc_hwmon_fan_auto(struct macsmc_hwmon *hwmon, ret = macsmc_hwmon_write_key(hwmon->smc, &fan->mode, MACSMC_FAN_MODE_AUTO); fan->manual = false; + fan->kernel = false; hwmon->fan.manual--; if (hwmon->fan.unlocked && !hwmon->fan.manual) { @@ -578,6 +581,8 @@ static int macsmc_hwmon_set_fan(struct macsmc_hwmon *hwmon, int channel, long va if (!ret) ret = macsmc_hwmon_fan_manual(hwmon, fan); if (!ret) { + /* Userspace takes the fan over from a kernel policy. */ + fan->kernel = false; ret = macsmc_hwmon_write_key(hwmon->smc, &fan->set, clamp(val, min, max)); if (ret >= 0) return 0; @@ -587,6 +592,102 @@ static int macsmc_hwmon_set_fan(struct macsmc_hwmon *hwmon, int channel, long va return ret; } +/** + * macsmc_hwmon_fans_request() - set the fans for a kernel thermal policy + * @dev: the bound macsmc-hwmon device; the caller holds its device lock + * @permille: each fan's target as a share of its minimum-to-maximum range; + * 0 hands the fans this interface set back to the SMC + * @min_step_rpm: a fan already set by this interface whose target is closer + * than this to the new one keeps its target + * + * The fans go through the same manual mode as a userspace request, so the + * mode key has one owner: fans userspace holds are left alone, and every fan + * returns to the SMC at suspend, shutdown and unbind. Only fans whose device + * tree node names their mode, target, minimum and maximum keys take part. + * + * Return: the first such fan's target in rpm (0 once handed back), -ENODEV + * without such a fan, -EBUSY while stopped, or the first error. + */ +int macsmc_hwmon_fans_request(struct device *dev, unsigned int permille, + unsigned int min_step_rpm) +{ + struct macsmc_hwmon *hwmon = dev_get_drvdata(dev); + long first = -ENODEV; + int ret = 0; + u32 i; + + if (!hwmon) + return -ENODEV; + permille = min(permille, 1000U); + + guard(mutex)(&hwmon->fan.lock); + + if (permille && hwmon->fan.stopped) + return -EBUSY; + + for (i = 0; i < hwmon->fan.count; i++) { + struct macsmc_hwmon_fan *fan = &hwmon->fan.fans[i]; + long min, max, now, target; + int err; + + if (!fan->mode.macsmc_key || + (fan->attrs & (HWMON_F_TARGET | HWMON_F_MIN | HWMON_F_MAX)) != + (HWMON_F_TARGET | HWMON_F_MIN | HWMON_F_MAX)) + continue; + if (fan->manual && !fan->kernel) + continue; + + if (!permille) { + if (first < 0) + first = 0; + err = macsmc_hwmon_fan_auto(hwmon, fan); + if (err && !ret) + ret = err; + continue; + } + + err = macsmc_hwmon_read_key(hwmon->smc, &fan->min, 1, &min); + if (!err) + err = macsmc_hwmon_read_key(hwmon->smc, &fan->max, 1, &max); + if (!err && (min <= 0 || max < min)) + err = -EIO; + if (err) { + dev_warn_ratelimited(hwmon->dev, "%s: no fan range for a kernel request (%d)\n", + fan->label, err); + if (!ret) + ret = err; + continue; + } + target = min + (max - min) * permille / 1000; + if (first < 0) + first = target; + + if (fan->kernel && + !macsmc_hwmon_read_key(hwmon->smc, &fan->set, 1, &now) && + abs(now - target) < (long)min_step_rpm) + continue; + + err = macsmc_hwmon_fan_manual(hwmon, fan); + if (!err) { + fan->kernel = true; + err = macsmc_hwmon_write_key(hwmon->smc, &fan->set, target); + if (err > 0) + err = 0; + } + if (err) { + /* -EOPNOTSUPP: the fan is in system mode and this SMC has no unlock key. */ + dev_warn_ratelimited(hwmon->dev, "%s: kernel fan request refused (%d)\n", + fan->label, err); + macsmc_hwmon_fan_auto(hwmon, fan); + if (!ret) + ret = err; + } + } + + return ret ? ret : first; +} +EXPORT_SYMBOL_GPL(macsmc_hwmon_fans_request); + static int macsmc_hwmon_write_fan(struct device *dev, u32 attr, int channel, long val) { diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index e9929282d0f78..7e01a1230d995 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -25,6 +25,7 @@ #include #include #include +#include #include #include #include @@ -305,6 +306,21 @@ struct apple_dart_fw_root { u64 *accepted; }; +/* + * The subtrees behind the slots firmware holds in a four-level locked root + * (fw_mirror), recorded when the root is mapped: per slot, the root entry and + * a mapping of its mid-level table, and per mid entry the entry and a mapping + * of its leaf table. Leaf entries are read live. A table entry that no longer + * matches what was recorded resolves nothing. Only lookups use it: Linux + * still never publishes into those slots. + */ +struct apple_dart_fw_tree { + u64 *entry; + u64 **mid; + u64 **mid_entry; + u64 ***leaf; +}; + /* * Private structure associated with each DART device. * @@ -346,6 +362,15 @@ struct apple_dart { u32 power_retained : 1; /* Locked roots may hold firmware slots that mirror Linux mappings. */ u32 fw_mirror : 1; + /* + * fw_mirror's four-level locked roots hold firmware's own subtrees, which + * are recorded and reserved (apple_dart_snapshot_fw_tree()). Set only for + * apple,t6050-dart, where it was tested (the DCP's log and crash log). Before + * widening it, check on T6041/T8140/T8142 whether any locked stream is + * four-level and what its foreign slots hold: the reservation moves DMA + * placement for every client of that stream. + */ + u32 fw_tree : 1; u32 fw_handoff : 1; u32 tunnel_state_saved : 1; u32 tunnel_state_restored : 1; @@ -437,6 +462,8 @@ struct apple_dart { u64 *locked_owned[DART_MAX_STREAMS][DART_MAX_TTBR]; /* Leaf tables behind the root slots firmware held when we attached. */ struct apple_dart_fw_root *locked_fw[DART_MAX_STREAMS][DART_MAX_TTBR]; + /* Firmware's subtrees in a four-level locked root (TTBR 0 only). */ + struct apple_dart_fw_tree *locked_fw_tree[DART_MAX_STREAMS]; /* * First DVA of the window of the domain using each locked stream. The * window can be the client's own (apple,dma-range on the client node) @@ -904,15 +931,22 @@ static bool apple_dart_fw_handoff_enabled(const struct apple_dart *dart) handoff == 1; } +/* A 16K table page, aligned and inside the output address space. */ +static bool apple_dart_fw_table_page(const struct apple_dart *dart, + phys_addr_t phys, phys_addr_t *end) +{ + return dart->pgsize == SZ_16K && IS_ALIGNED(phys, dart->pgsize) && + !check_add_overflow(phys, (phys_addr_t)dart->pgsize - 1, end) && + dart->oas < 64 && *end < BIT_ULL(dart->oas); +} + static bool apple_dart_fw_table_ram(const struct apple_dart *dart, phys_addr_t phys) { unsigned long first, last, pfn; phys_addr_t end; - if (dart->pgsize != SZ_16K || !IS_ALIGNED(phys, dart->pgsize) || - check_add_overflow(phys, (phys_addr_t)dart->pgsize - 1, &end) || - dart->oas >= 64 || end >= BIT_ULL(dart->oas)) + if (!apple_dart_fw_table_page(dart, phys, &end)) return false; first = PHYS_PFN(phys); last = PHYS_PFN(end); @@ -922,14 +956,41 @@ static bool apple_dart_fw_table_ram(const struct apple_dart *dart, return true; } -static bool apple_dart_fw_table_decode(const struct apple_dart *dart, - u64 pte, phys_addr_t *phys) +/* A valid table entry with no bits but the next table's address. */ +static bool apple_dart_fw_table_pte(const struct apple_dart *dart, u64 pte, + phys_addr_t *phys) { if (!(pte & APPLE_DART_PTE_VALID) || (pte & ~(GENMASK_ULL(37, 10) | APPLE_DART_PTE_VALID))) return false; *phys = apple_dart_pte_to_paddr(dart, pte); - return apple_dart_fw_table_ram(dart, *phys); + return true; +} + +static bool apple_dart_fw_table_decode(const struct apple_dart *dart, + u64 pte, phys_addr_t *phys) +{ + return apple_dart_fw_table_pte(dart, pte, phys) && + apple_dart_fw_table_ram(dart, *phys); +} + +/* + * A table page of firmware's four-level subtrees, which is mapped write-back: + * reserved RAM as for apple_dart_fw_table_ram(), or memory no resource + * claims. T6050's boot loader places these tables outside the memory node, + * so they are not RAM to Linux. An entry naming Linux's own RAM or a + * device's registers is refused, and its DVAs resolve nothing. + */ +static bool apple_dart_fw_tree_table(const struct apple_dart *dart, + phys_addr_t phys) +{ + phys_addr_t end; + + if (apple_dart_fw_table_ram(dart, phys)) + return true; + return apple_dart_fw_table_page(dart, phys, &end) && + region_intersects(phys, dart->pgsize, IORESOURCE_MEM, + IORES_DESC_NONE) == REGION_DISJOINT; } static bool apple_dart_fw_slot_owned(const struct apple_dart_fw_root *fw, @@ -1156,6 +1217,118 @@ apple_dart_snapshot_fw_root(struct apple_dart *dart, int sid, const u64 *live) return fw; } +static void apple_dart_free_fw_tree(struct apple_dart *dart, + struct apple_dart_fw_tree *tree) +{ + size_t i, j, entries = dart->pgsize / sizeof(u64); + + if (!tree) + return; + /* Any of the four arrays may be missing after a failed allocation. */ + for (i = 0; i < entries; i++) { + if (tree->leaf && tree->leaf[i]) { + for (j = 0; j < entries; j++) + if (tree->leaf[i][j]) + memunmap(tree->leaf[i][j]); + kfree(tree->leaf[i]); + } + if (tree->mid_entry) + kfree(tree->mid_entry[i]); + if (tree->mid && tree->mid[i]) + memunmap(tree->mid[i]); + } + kfree(tree->leaf); + kfree(tree->mid_entry); + kfree(tree->mid); + kfree(tree->entry); + kfree(tree); +} + +/* + * Record firmware's subtrees in the four-level root @live of @sid, so that + * firmware-placed buffers (an RTKit co-processor's shared segments, which + * it asks the AP to map at its own DVA) resolve through iova_to_phys. A + * table that cannot be mapped, or that apple_dart_fw_tree_table() refuses, + * is left out; its DVAs resolve nothing. A slot can map up to one leaf table + * per mid-level entry (2048 with 16K pages); firmware holds a few slots. + */ +static struct apple_dart_fw_tree * +apple_dart_snapshot_fw_tree(struct apple_dart *dart, int sid, const u64 *live) +{ + size_t i, j, entries = dart->pgsize / sizeof(u64); + struct apple_dart_fw_tree *tree; + + if (!dart->fw_tree || !dart->fw_mirror || dart->fw_handoff || + dart->hw->fmt != APPLE_DART2 || + !dart->hw->tcr_4level || + !(apple_dart_readl(dart, DART_TCR(dart, sid)) & dart->hw->tcr_4level)) + return NULL; + + tree = kzalloc(sizeof(*tree), GFP_KERNEL); + if (!tree) + return NULL; + tree->entry = kcalloc(entries, sizeof(*tree->entry), GFP_KERNEL); + tree->mid = kcalloc(entries, sizeof(*tree->mid), GFP_KERNEL); + tree->mid_entry = kcalloc(entries, sizeof(*tree->mid_entry), GFP_KERNEL); + tree->leaf = kcalloc(entries, sizeof(*tree->leaf), GFP_KERNEL); + if (!tree->entry || !tree->mid || !tree->mid_entry || !tree->leaf) { + apple_dart_free_fw_tree(dart, tree); + return NULL; + } + + for (i = 0; i < entries; i++) { + u64 pte = READ_ONCE(live[i]); + phys_addr_t phys; + + if (!apple_dart_fw_table_pte(dart, pte, &phys) || + !apple_dart_fw_tree_table(dart, phys)) + continue; + tree->mid_entry[i] = kcalloc(entries, sizeof(u64), GFP_KERNEL); + tree->leaf[i] = kcalloc(entries, sizeof(u64 *), GFP_KERNEL); + tree->mid[i] = memremap(phys, dart->pgsize, MEMREMAP_WB); + if (!tree->mid_entry[i] || !tree->leaf[i] || !tree->mid[i]) + continue; + tree->entry[i] = pte; + for (j = 0; j < entries; j++) { + u64 mid = READ_ONCE(tree->mid[i][j]); + + if (!apple_dart_fw_table_pte(dart, mid, &phys) || + !apple_dart_fw_tree_table(dart, phys)) + continue; + tree->leaf[i][j] = memremap(phys, dart->pgsize, MEMREMAP_WB); + if (tree->leaf[i][j]) + tree->mid_entry[i][j] = mid; + } + } + return tree; +} + +/* What firmware's recorded four-level subtree translates @addr to, or 0. */ +static phys_addr_t apple_dart_fw_tree_phys(const struct apple_dart *dart, + const struct apple_dart_fw_tree *tree, + const u64 *live, u64 addr, int *prot_out) +{ + u32 pgshift = ilog2(dart->pgsize); + u32 bits = ilog2(dart->pgsize / sizeof(u64)); + u64 mask = BIT_ULL(bits) - 1; + u64 slot = addr >> (pgshift + 2 * bits); + u64 mid = (addr >> (pgshift + bits)) & mask; + u64 page = (addr >> pgshift) & mask; + phys_addr_t phys; + int prot; + + if (!tree || !live || slot > mask || !tree->entry[slot] || + READ_ONCE(live[slot]) != tree->entry[slot] || + !tree->mid_entry[slot][mid] || + READ_ONCE(tree->mid[slot][mid]) != tree->mid_entry[slot][mid] || + !apple_dart_fw_leaf_decode(READ_ONCE(tree->leaf[slot][mid][page]), + addr, &phys, &prot)) + return 0; + if (prot_out) + *prot_out = prot; + return phys; +} + static int apple_dart_fw_invalidate(struct apple_dart_stream_map *stream_map, u64 lo, u64 hi); @@ -1166,6 +1339,7 @@ apple_dart_hw_map_locked_ttbr(struct apple_dart_stream_map *stream_map, u8 idx) int sid; for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) { + struct apple_dart_fw_tree *tree = NULL; struct apple_dart_fw_root *fw; unsigned long flags; u32 ttbr; @@ -1212,10 +1386,14 @@ apple_dart_hw_map_locked_ttbr(struct apple_dart_stream_map *stream_map, u8 idx) devm_memunmap(dart->dev, l1_tbl); return PTR_ERR(fw); } + if (!idx) + tree = apple_dart_snapshot_fw_tree(dart, sid, l1_tbl); spin_lock_irqsave(&dart->lock, flags); dart->locked_owned[sid][idx] = owned; dart->locked_ttbr[sid][idx] = l1_tbl; dart->locked_fw[sid][idx] = fw; + if (!idx) + dart->locked_fw_tree[sid] = tree; spin_unlock_irqrestore(&dart->lock, flags); } @@ -1237,6 +1415,7 @@ apple_dart_hw_unmap_locked_ttbr(struct apple_dart_stream_map *stream_map, u8 idx for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) { u64 *live = dart->locked_ttbr[sid][idx]; u64 *owned = dart->locked_owned[sid][idx]; + struct apple_dart_fw_tree *tree = NULL; struct apple_dart_fw_root *fw; unsigned long flags; bool first = false; @@ -1261,6 +1440,10 @@ apple_dart_hw_unmap_locked_ttbr(struct apple_dart_stream_map *stream_map, u8 idx fw = dart->locked_fw[sid][idx]; dart->locked_fw[sid][idx] = NULL; dart->locked_ttbr[sid][idx] = NULL; + if (!idx) { + tree = dart->locked_fw_tree[sid]; + dart->locked_fw_tree[sid] = NULL; + } spin_unlock_irqrestore(&dart->lock, flags); /* Once per stream: the error is never cleared. */ if (first) @@ -1268,6 +1451,7 @@ apple_dart_hw_unmap_locked_ttbr(struct apple_dart_stream_map *stream_map, u8 idx "stream %d: TLB invalidate failed retiring its root (%d); keeping the page tables of its domains until reboot\n", sid, err); apple_dart_free_fw_root(dart, fw); + apple_dart_free_fw_tree(dart, tree); devm_memunmap(dart->dev, live); devm_kfree(dart->dev, owned); dart->locked_owned[sid][idx] = NULL; @@ -2439,11 +2623,18 @@ static int apple_dart_check_fw_map(struct apple_dart_domain *domain, u64 dva, return 0; } +/* + * What firmware's own retained translations in @domain map @address to, or 0. + * With @prot_out, also the protection every stream grants (IOMMU_READ and + * IOMMU_WRITE as the leaves say). + */ static phys_addr_t -apple_dart_inherited_phys(struct apple_dart_domain *domain, dma_addr_t address) +apple_dart_inherited_phys_prot(struct apple_dart_domain *domain, dma_addr_t address, + int *prot_out) { struct apple_dart_atomic_stream_map *map; phys_addr_t result = 0; + int granted = IOMMU_READ | IOMMU_WRITE | IOMMU_CACHE; int i, sid; for_each_stream_map(i, domain, map) { @@ -2453,31 +2644,90 @@ apple_dart_inherited_phys(struct apple_dart_domain *domain, dma_addr_t address) size_t page = (address / dart->pgsize) % entries; size_t slot = address / dart->pgsize / entries; - if (!dart->locked || (!dart->fw_handoff && !dart->fw_mirror) || slot >= entries) + if (!dart->locked || (!dart->fw_handoff && !dart->fw_mirror)) return 0; spin_lock_irqsave(&dart->lock, flags); for (sid = 0; sid < dart->num_streams; sid++) { + const struct apple_dart_fw_tree *tree; struct apple_dart_fw_root *fw; - phys_addr_t phys; + phys_addr_t phys = 0; int prot; if (!apple_dart_domain_uses_fw(domain, i, map, sid)) continue; + tree = dart->locked_fw_tree[sid]; fw = dart->locked_fw[sid][0]; - if (!apple_dart_fw_slot_owned(fw, dart->locked_ttbr[sid][0], slot) || - !apple_dart_fw_leaf_decode(READ_ONCE(fw->leaf[slot][page]), - address, &phys, &prot) || - (result && result != phys)) { + prot = 0; + if (tree) + phys = apple_dart_fw_tree_phys(dart, tree, + dart->locked_ttbr[sid][0], + address, &prot); + else if (slot >= entries || + !apple_dart_fw_slot_owned(fw, dart->locked_ttbr[sid][0], slot) || + !apple_dart_fw_leaf_decode(READ_ONCE(fw->leaf[slot][page]), + address, &phys, &prot)) + phys = 0; + if (!phys || (result && result != phys)) { spin_unlock_irqrestore(&dart->lock, flags); return 0; } result = phys; + granted &= prot; } spin_unlock_irqrestore(&dart->lock, flags); } + if (prot_out) + *prot_out = result ? granted : 0; return result; } +static phys_addr_t +apple_dart_inherited_phys(struct apple_dart_domain *domain, dma_addr_t address) +{ + return apple_dart_inherited_phys_prot(domain, address, NULL); +} + +/* + * Resolves a DVA a co-processor's firmware names in a request of its own, for + * a host copy to or from that buffer. Only firmware's retained mappings count: + * a DVA Linux mapped itself (a framebuffer, an imported or DMA_TO_DEVICE + * buffer) is refused, so the host never writes what the device may only read. + * @write requires the leaf to grant writes. -EACCES when refused, -ENXIO when + * unmapped. + */ +int apple_dart_fw_remote(struct device *dev, dma_addr_t iova, bool write, + phys_addr_t *phys) +{ + struct iommu_domain *domain = iommu_get_domain_for_dev(dev); + struct apple_dart_domain *dart_domain; + struct io_pgtable_ops *ops; + struct io_pgtable_cfg *cfg; + dma_addr_t address; + phys_addr_t result; + int prot; + + if (!domain || domain->owner != &apple_dart_iommu_ops) + return -ENODEV; + dart_domain = to_dart_domain(domain); + ops = dart_domain->pgtbl_ops; + if (dart_domain->dcp || !ops) + return -ENODEV; + address = (iova + dart_domain->dma_offset) & dart_domain->mask; + if (ops->iova_to_phys(ops, address)) + return -EACCES; + cfg = &io_pgtable_ops_to_pgtable(ops)->cfg; + if (cfg->apple_dart_cfg.n_levels != 3 && cfg->apple_dart_cfg.n_levels != 4) + return -ENXIO; + result = apple_dart_inherited_phys_prot(dart_domain, address, &prot); + if (!result) + return -ENXIO; + if (!(prot & IOMMU_READ) || (write && !(prot & IOMMU_WRITE))) + return -EACCES; + *phys = result; + return 0; +} +EXPORT_SYMBOL_GPL(apple_dart_fw_remote); + static phys_addr_t apple_dart_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova) { @@ -2497,7 +2747,7 @@ static phys_addr_t apple_dart_iova_to_phys(struct iommu_domain *domain, if (phys) return phys; cfg = &io_pgtable_ops_to_pgtable(ops)->cfg; - if (cfg->apple_dart_cfg.n_levels != 3) + if (cfg->apple_dart_cfg.n_levels != 3 && cfg->apple_dart_cfg.n_levels != 4) return 0; /* Existing firmware translations need no replacement software mapping. */ return apple_dart_inherited_phys(dart_domain, address); @@ -3734,7 +3984,12 @@ static void apple_dart_get_fw_resv_regions(struct device *dev, struct apple_dart *dart = stream->dart; size_t entries = dart->pgsize / sizeof(u64); - if (!dart->fw_handoff || entries > DART_MAX_ROOT_ENTRIES) + /* + * Only the handoff and fw_tree reserve anything: leave the other + * fw_mirror DARTs (T6041, T8140, T8142) untouched, TCR reads included. + */ + if ((!dart->fw_handoff && !dart->fw_tree) || + entries > DART_MAX_ROOT_ENTRIES) continue; for_each_set_bit(sid, stream->sidmap, dart->num_streams) { for (idx = 0; idx < dart->hw->ttbr_count; idx++) { @@ -3758,6 +4013,13 @@ static void apple_dart_get_fw_resv_regions(struct device *dev, __set_bit(slot, slots); spin_unlock_irqrestore(&dart->lock, flags); + /* + * A three-level fw_mirror slot may repeat firmware's + * pages (apple_dart_check_fw_map()); a four-level one + * is firmware's alone, so keep the allocator out of it. + */ + if (!dart->fw_handoff && !(four && dart->fw_tree)) + continue; if (!four && check_add_overflow(window, ((u64)idx << (ilog2(dart->pgsize) + 2 * ilog2(entries))), &window)) @@ -4807,9 +5069,14 @@ static int apple_dart_probe(struct platform_device *pdev) } /* Reserved table-page lifetime requires the matched three-level loader. */ dart->fw_handoff = apple_dart_fw_handoff_enabled(dart); - /* Preserve our retained-root path when no reserved-table handoff exists. */ + /* + * Preserve our retained-root path when no reserved-table handoff exists. + * T6050's boot loader leaves every display DART locked on four-level + * roots of its own (dart-dcpextN SID 23, dart-dispextN SIDs 0 and 16). + */ dart->fw_mirror = dart->locked && !dart->fw_handoff && (of_device_is_compatible(dev->of_node, "apple,t6041-dart") || + of_device_is_compatible(dev->of_node, "apple,t6050-dart") || of_device_is_compatible(dev->of_node, "apple,t8140-dart") || of_device_is_compatible(dev->of_node, "apple,t8142-dart")); /* @@ -4844,6 +5111,8 @@ static int apple_dart_probe(struct platform_device *pdev) of_node_put(iommu); of_node_put(mtp); } + dart->fw_tree = dart->fw_mirror && + of_device_is_compatible(dev->of_node, "apple,t6050-dart"); if (apple_t8132_live_mtp() && of_machine_is_compatible("apple,t8132") && res->start == 0x394800000ULL && dart->hw->type == DART_T8110 && !dart->locked && !tunneled && dart->supports_bypass && diff --git a/include/linux/mfd/macsmc.h b/include/linux/mfd/macsmc.h index fbc047b00afd8..604c54b0b3274 100644 --- a/include/linux/mfd/macsmc.h +++ b/include/linux/mfd/macsmc.h @@ -267,6 +267,23 @@ APPLE_SMC_TYPE_OPS(s32) APPLE_SMC_TYPE_OPS(s16) APPLE_SMC_TYPE_OPS(s8) +struct device; + +/* + * The SMC hwmon cell's fan control for a kernel thermal policy (a GPU + * governor): see drivers/hwmon/macsmc-hwmon.c. + */ +#if IS_REACHABLE(CONFIG_SENSORS_MACSMC_HWMON) +int macsmc_hwmon_fans_request(struct device *dev, unsigned int permille, + unsigned int min_step_rpm); +#else +static inline int macsmc_hwmon_fans_request(struct device *dev, unsigned int permille, + unsigned int min_step_rpm) +{ + return -ENODEV; +} +#endif + static inline int apple_smc_read_flag(struct apple_smc *smc, smc_key key, bool *flag) { u8 val; diff --git a/include/linux/soc/apple/dart.h b/include/linux/soc/apple/dart.h index 10d05a7cc042b..9cf5f99f8b8ea 100644 --- a/include/linux/soc/apple/dart.h +++ b/include/linux/soc/apple/dart.h @@ -153,12 +153,19 @@ int apple_dart_save_tunnel_state(struct device *dev); int apple_dart_restore_tunnel_state(struct device *dev); /* Registered J713 managed radio provider, with no owned valid root or fault. */ int apple_dart_radio_check_idle(struct device *dev); +/* A firmware-owned buffer's address for a host copy; Linux's own mappings refused. */ +int apple_dart_fw_remote(struct device *dev, dma_addr_t iova, bool write, phys_addr_t *phys); #else static inline int apple_dart_quiesce_commands(struct device *dev) { return 0; } static inline int apple_dart_resume_commands(struct device *dev) { return 0; } static inline int apple_dart_save_tunnel_state(struct device *dev) { return -EOPNOTSUPP; } static inline int apple_dart_restore_tunnel_state(struct device *dev) { return -EOPNOTSUPP; } static inline int apple_dart_radio_check_idle(struct device *dev) { return -EOPNOTSUPP; } +static inline int apple_dart_fw_remote(struct device *dev, dma_addr_t iova, bool write, + phys_addr_t *phys) +{ + return -EOPNOTSUPP; +} #endif #endif diff --git a/include/uapi/drm/asahi_drm.h b/include/uapi/drm/asahi_drm.h index 55092da885071..3a0bf7ae28b58 100644 --- a/include/uapi/drm/asahi_drm.h +++ b/include/uapi/drm/asahi_drm.h @@ -371,11 +371,11 @@ enum drm_asahi_feature { DRM_ASAHI_FEATURE_BOUNDED_AS_BUILD = (1UL) << 16, /* - * Bits 17 to 28 belong to the M5 (G17G/G17S) runtime, which reports - * them; this kernel never does. They are listed so that the numbers are - * not given to anything else, and userspace reads every kernel's - * features the same way. Bits 25 and 27 are also reserved for that - * runtime. + * Bits 17 to 28 belong to the M5 (G17G/G17S) command set. This kernel + * reports them only on M5 GPUs, and never bits 24 and 25 (see + * DRM_ASAHI_FEATURE_RENDER_STAGE_PREFIXES). They are listed so that the + * numbers are not given to anything else, and userspace reads every + * kernel's features the same way. */ /** @DRM_ASAHI_FEATURE_RENDER_RESOURCES: M5: per-program render storage. */ @@ -404,16 +404,33 @@ enum drm_asahi_feature { /** * @DRM_ASAHI_FEATURE_COMPUTE_STAGE_BARRIERS: M5: a compute prefix for a - * render's tiling half. + * render's tiling half, in a shorter command form. Retired with + * DRM_ASAHI_FEATURE_RENDER_STAGE_PREFIXES, for the same reason. */ DRM_ASAHI_FEATURE_COMPUTE_STAGE_BARRIERS = (1UL) << 24, + /** + * @DRM_ASAHI_FEATURE_RENDER_STAGE_PREFIXES: M5: render commands accept + * DRM_ASAHI_RENDER_STAGE_PREFIXES in a shorter command form. Retired: no + * kernel may report this bit or bit 24 again. Userspace that sees either + * sends staged renders of 352 or 368 bytes with prefix fields at bytes + * 336 to 367, which struct drm_asahi_cmd_render gives to its ray-query + * stages. Staged renders use DRM_ASAHI_FEATURE_RENDER_STAGE_PREFIXES_V2. + */ + DRM_ASAHI_FEATURE_RENDER_STAGE_PREFIXES = (1UL) << 25, + /** * @DRM_ASAHI_FEATURE_M5_STENCIL_MSAA_COMPRESSION: M5: compressed * multisample stencil. */ DRM_ASAHI_FEATURE_M5_STENCIL_MSAA_COMPRESSION = (1UL) << 26, + /** + * @DRM_ASAHI_FEATURE_WHOLE_ATTACHMENT_RENDERS: M5: render commands accept + * DRM_ASAHI_RENDER_WHOLE_ATTACHMENTS. + */ + DRM_ASAHI_FEATURE_WHOLE_ATTACHMENT_RENDERS = (1UL) << 27, + /** @DRM_ASAHI_FEATURE_COMPUTE_RESOURCES: M5: per-dispatch storage. */ DRM_ASAHI_FEATURE_COMPUTE_RESOURCES = (1UL) << 28, @@ -424,11 +441,18 @@ enum drm_asahi_feature { */ DRM_ASAHI_FEATURE_CUMULATIVE_SIGNALS = (1UL) << 29, + /** + * @DRM_ASAHI_FEATURE_COMPUTE_NO_QUERY_ACCESS: Compute commands accept + * DRM_ASAHI_COMPUTE_NO_QUERY_ACCESS as an ordering hint. + */ + DRM_ASAHI_FEATURE_COMPUTE_NO_QUERY_ACCESS = (1UL) << 30, + /* - * Bit 30 is used by the M5 runtime's userspace. Bits 31 and 32 belong - * to the G18 (M6, T8152) backend, which alone reports them. Bit 32 is - * DRM_ASAHI_FEATURE_G18_NATIVE_TILE_SAMPLES, defined after this enum so - * that the enum keeps its 32-bit type. + * Bit 30 above is reported by the M5 command set. Bits 31 and 32 + * belong to the G18 (M6, T8152) backend, which alone reports them. + * Bit 32 is DRM_ASAHI_FEATURE_G18_NATIVE_TILE_SAMPLES, defined after + * this enum so that the enum keeps its 32-bit type. Bit 33, + * DRM_ASAHI_FEATURE_RENDER_STAGE_PREFIXES_V2, is defined there too. */ /** @@ -450,6 +474,21 @@ enum drm_asahi_feature { */ #define DRM_ASAHI_FEATURE_G18_NATIVE_TILE_SAMPLES ((1ULL) << 32) +/* + * DRM_ASAHI_FEATURE_RENDER_STAGE_PREFIXES_V2: M5: render commands accept + * DRM_ASAHI_RENDER_STAGE_PREFIXES as the complete struct + * drm_asahi_cmd_render_m5. A feature bit like those of enum + * drm_asahi_feature, defined here like bit 32. + */ +#define DRM_ASAHI_FEATURE_RENDER_STAGE_PREFIXES_V2 ((1ULL) << 33) + +/* + * DRM_ASAHI_FEATURE_COMPUTE_VISIBLE_COMPLETION: M5: compute commands accept + * DRM_ASAHI_COMPUTE_VISIBLE_COMPLETION. A feature bit like those of enum + * drm_asahi_feature, defined here like bit 32. + */ +#define DRM_ASAHI_FEATURE_COMPUTE_VISIBLE_COMPLETION ((1ULL) << 34) + #define DRM_ASAHI_PARAM_GROUP_VM_STATUS 1 #define DRM_ASAHI_PARAM_GROUP_QUEUE_LIMITS 2 @@ -1044,6 +1083,87 @@ struct drm_asahi_sync { */ #define DRM_ASAHI_BARRIER_NONE (0xFFFFu) +/** + * struct drm_asahi_compute_program - Storage of one linked compute dispatch. + * + * SET_COMPUTE_RESOURCES replaces the list for subsequent compute commands. + * Include each distinct dispatch profile in the command stream. The kernel + * combines their requirements before publishing work. All sizes are bytes. + */ +struct drm_asahi_compute_program { + /** @primary_bytes: Per-thread primary storage. */ + __u32 primary_bytes; + /** @secondary_bytes: Per-thread secondary storage. */ + __u32 secondary_bytes; + /** @linked_bytes: Additional primary storage after linking. */ + __u32 linked_bytes; + /** @minimum_primary_bytes: Minimum primary storage across variants. */ + __u32 minimum_primary_bytes; + /** @threadgroup_bytes: Storage shared by a workgroup. */ + __u32 threadgroup_bytes; + /** @threads: Number of invocations in a workgroup. */ + __u32 threads; + /** @flags: Reserved; must be zero. */ + __u32 flags; +}; + +/** + * struct drm_asahi_render_program - Storage of one linked render program. + * + * SET_RENDER_RESOURCES replaces the list for subsequent renders in this submit. + * Include every vertex and fragment program, including background and end-of-tile + * programs. Sizes describe the final linked program, in bytes. Each entry is + * evaluated separately before taking the maximum demand across the pass. + */ +struct drm_asahi_render_program { + /** @stage: 0 for vertex, 1 for fragment (including background/end-of-tile). */ + __u32 stage; + /** @primary_bytes: Primary per-thread storage after linking. */ + __u32 primary_bytes; + /** @secondary_bytes: Secondary per-thread storage after linking. */ + __u32 secondary_bytes; + /** @linked_bytes: Additional primary storage from linked programs. */ + __u32 linked_bytes; + /** @minimum_primary_bytes: Minimum primary storage across linked variants. */ + __u32 minimum_primary_bytes; + /** @auxiliary_count: Fragment auxiliary entries; zero for vertex programs. */ + __u32 auxiliary_count; + /** @threadgroup_bytes: Fragment threadgroup storage; zero for vertex. */ + __u32 threadgroup_bytes; + /** @imageblock_bytes: Fragment imageblock storage; zero for vertex. */ + __u32 imageblock_bytes; + /** @multisample_bytes: Fragment multisample storage; zero for vertex. */ + __u32 multisample_bytes; + /** @flags: MBZ. */ + __u32 flags; +}; +/** Version 2 linked render storage. Version 1 remains exactly 40 bytes. */ +#define DRM_ASAHI_RENDER_PROGRAM_SCS (1U << 0) +#define DRM_ASAHI_RENDER_PROGRAM_MESH (1U << 1) +#define DRM_ASAHI_RENDER_PROGRAM_OBJECT (1U << 2) +/* With OBJECT: the object program is an API task stage. The kernel binds the + * TA mesh rendering buffer (the backing of the object payload address) to + * such kicks only. Requires DRM_ASAHI_FEATURE_TASK_MESH_STORAGE. */ +#define DRM_ASAHI_RENDER_PROGRAM_TASK (1U << 3) +struct drm_asahi_render_program_v2 { + struct drm_asahi_render_program legacy; + /** @threads: Mesh/object workgroup invocation count; zero otherwise. */ + __u32 threads; + /** @usage_flags: DRM_ASAHI_RENDER_PROGRAM_* bits. */ + __u32 usage_flags; + /** @scs_layout: Packed depth/levels/cliques/user/driver SCS layout. */ + __u64 scs_layout; + /** @auxiliary_allocation: The stage's auxiliary allocation word. */ + __u32 auxiliary_allocation; + __u32 reserved; + __u64 intersection_pool; + __u64 intersection_pool_bytes; + __u32 intersection_first; + __u32 intersection_count; + __u64 intersection_default_program; + __u64 intersection_default_control; +}; + /** * struct drm_asahi_cmd_header - Top level command structure * @@ -1283,6 +1403,38 @@ enum drm_asahi_render_flags { * Bits 21 to 24 are used by the M5 runtime's userspace. */ + /** + * @DRM_ASAHI_RENDER_SHADER_WRITES: A graphics program in this command + * writes external memory, including buffer/image stores or atomic RMW. + * Ordinary raster outputs and compiler-private transport do not count. + * Requires DRM_ASAHI_FEATURE_RENDER_SHADER_WRITES. Omission retains the + * existing encoding, allowing older userspace to submit unchanged. + */ + DRM_ASAHI_RENDER_SHADER_WRITES = (1U << 21), + + /** + * @DRM_ASAHI_RENDER_STAGE_PREFIXES: The tiling half waits only on the + * render and compute prefixes in struct drm_asahi_render_stage_prefixes + * (ta_vdm_barrier, ta_prior_vdm, ta_cdm_barrier and ta_prior_cdm), each + * no later than the command's own prefix of that subqueue; the fragment + * half waits on the whole ones. Earlier renders in the tiling prefix are + * waited on to completion. The payload must be the complete struct + * drm_asahi_cmd_render_m5. Not combined with + * DRM_ASAHI_RENDER_FRAGMENT_BARRIERS. Requires + * DRM_ASAHI_FEATURE_RENDER_STAGE_PREFIXES_V2. + */ + DRM_ASAHI_RENDER_STAGE_PREFIXES = (1U << 23), + + /** + * @DRM_ASAHI_RENDER_WHOLE_ATTACHMENTS: The render area covers the full + * extent of every attachment, so the render's tile stores write no pixel + * outside it. The kernel need not order such a render against another + * one that writes the same attachments without a dependency between + * them: any overlap is the application's own race. Requires + * DRM_ASAHI_FEATURE_WHOLE_ATTACHMENT_RENDERS. + */ + DRM_ASAHI_RENDER_WHOLE_ATTACHMENTS = (1U << 24), + /** * @DRM_ASAHI_RENDER_G18_RESOURCES: G18 only; requires * DRM_ASAHI_FEATURE_G18_GRAPHICS_SCRATCH. The payload is a complete @@ -1672,6 +1824,66 @@ struct drm_asahi_cmd_render { struct drm_asahi_render_ray_query_stage ray_query[2]; }; +/** + * struct drm_asahi_render_stage_prefixes - Tiling-half prefixes of an M5 render + * + * Carried by render commands with DRM_ASAHI_RENDER_STAGE_PREFIXES, after the + * complete struct drm_asahi_cmd_render (see struct drm_asahi_cmd_render_m5). + * The kernel reads it only with that flag; a render without the flag may + * carry it only as zeroes. + */ +struct drm_asahi_render_stage_prefixes { + /** + * @ta_cdm_barrier: Compute prefix of the tiling half, a command index like + * drm_asahi_cmd_header::cdm_barrier. + */ + __u16 ta_cdm_barrier; + + /** @reserved0: MBZ. */ + __u16 reserved0; + + /** @reserved1: MBZ. */ + __u32 reserved1; + + /** + * @ta_prior_cdm: Exact compute prefix of the tiling half, an ordinal like + * drm_asahi_cmd_render::prior_cdm. + */ + __u64 ta_prior_cdm; + + /** + * @ta_vdm_barrier: Render prefix of the tiling half, a command index like + * drm_asahi_cmd_header::vdm_barrier. + */ + __u16 ta_vdm_barrier; + + /** @reserved2: MBZ. */ + __u16 reserved2; + + /** @reserved3: MBZ. */ + __u32 reserved3; + + /** + * @ta_prior_vdm: Exact render prefix of the tiling half, an ordinal like + * drm_asahi_cmd_render::prior_vdm. + */ + __u64 ta_prior_vdm; +}; + +/** + * struct drm_asahi_cmd_render_m5 - M5 render payload with stage prefixes + * + * The payload of a render command with DRM_ASAHI_RENDER_STAGE_PREFIXES: the + * render command followed by its tiling-half prefixes. Every field of @legacy + * keeps its meaning. + */ +struct drm_asahi_cmd_render_m5 { + /** @legacy: The render command. */ + struct drm_asahi_cmd_render legacy; + /** @stage_prefixes: The tiling half's render and compute prefixes. */ + struct drm_asahi_render_stage_prefixes stage_prefixes; +}; + /** * struct drm_asahi_cmd_compute - Command to submit compute * @@ -1712,9 +1924,10 @@ struct drm_asahi_cmd_compute { struct drm_asahi_timestamp ts_end_copies[8]; /** - * @scs_layout: Packed G17P Gen1 shader-context scratch layout, or zero. - * Bits 0:15 contain stack depth, 16:23 levels, 24:31 cliques, - * 32:47 user bytes and 48:63 driver bytes. Non-G17P paths require zero. + * @scs_layout: Packed shader-context scratch layout on G17P (Gen1) and + * M5, or zero. Bits 0:15 contain stack depth, 16:23 levels, 24:31 + * cliques, 32:47 user bytes and 48:63 driver bytes. Other GPUs require + * zero. */ __u64 scs_layout; @@ -1736,7 +1949,13 @@ struct drm_asahi_cmd_compute { */ __u64 prior_cdm; - /** @cdm_auxiliary_allocation: G17P auxiliary allocation, zero or 8..32. */ + /** + * @cdm_auxiliary_allocation: M5: the per-core page count of the + * dispatch's shader-context scratch storage, which the hardware is + * given as is and the scratch pool covers. Zero, or 8..32 with + * @intersection_pool and shader-context scratch (@scs_layout). Other + * GPUs require zero. + */ __u32 cdm_auxiliary_allocation; /** @reserved: MBZ. */ __u32 reserved; @@ -1781,6 +2000,20 @@ enum drm_asahi_compute_flags { * Bits 16 and 17 are used by the M5 runtime's userspace. */ + /** + * @DRM_ASAHI_COMPUTE_USES_MATRIX: The command's dispatches use the + * hardware's matrix instructions. Requires DRM_ASAHI_FEATURE_COMPUTE_MATRIX. + */ + DRM_ASAHI_COMPUTE_USES_MATRIX = (1L << 16), + + /** + * @DRM_ASAHI_COMPUTE_NO_QUERY_ACCESS: The command neither copies nor + * resets queries, so it cannot observe timestamps the kernel publishes + * for earlier commands and need not wait for that publication. Requires + * DRM_ASAHI_FEATURE_COMPUTE_NO_QUERY_ACCESS. + */ + DRM_ASAHI_COMPUTE_NO_QUERY_ACCESS = (1L << 17), + /** * @DRM_ASAHI_COMPUTE_USES_TEXTURE_STATE: G18 only. The command's * dispatch binds texture or image state (a texture table, a sampled @@ -1796,6 +2029,15 @@ enum drm_asahi_compute_flags { * Any bytes after it are zero. */ DRM_ASAHI_COMPUTE_G18_RESOURCES = (1L << 19), + + /** + * @DRM_ASAHI_COMPUTE_VISIBLE_COMPLETION: M5: the command's control + * stream ends with a strong, memory-scoped cache flush, so its writes + * are visible once it completes. A compute command of another queue may + * then wait for it in firmware rather than on the host. Requires + * DRM_ASAHI_FEATURE_COMPUTE_VISIBLE_COMPLETION. + */ + DRM_ASAHI_COMPUTE_VISIBLE_COMPLETION = (1L << 20), }; /** diff --git a/rust/helpers/cpumask.c b/rust/helpers/cpumask.c index 5deced5b975e0..c3366c96822b4 100644 --- a/rust/helpers/cpumask.c +++ b/rust/helpers/cpumask.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 #include +#include __rust_helper void rust_helper_cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp) @@ -81,3 +82,9 @@ void rust_helper_free_cpumask_var(cpumask_var_t mask) free_cpumask_var(mask); } #endif + +__rust_helper +const struct cpumask *rust_helper_topology_cluster_cpumask(int cpu) +{ + return topology_cluster_cpumask(cpu); +} diff --git a/rust/helpers/workqueue.c b/rust/helpers/workqueue.c index 7cd3b000a5b6b..c79f38f98f9c2 100644 --- a/rust/helpers/workqueue.c +++ b/rust/helpers/workqueue.c @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 +#include #include __rust_helper void rust_helper_init_work_with_key(struct work_struct *work, @@ -20,3 +21,26 @@ rust_helper_alloc_ordered_workqueue(const char *name, unsigned int flags) { return alloc_ordered_workqueue("%s", flags, name); } + +__rust_helper struct workqueue_struct * +rust_helper_alloc_workqueue_flags(const char *name, unsigned int flags) +{ + return alloc_workqueue("%s", flags, 0, name); +} + +__rust_helper int rust_helper_workqueue_restrict(struct workqueue_struct *wq, + const struct cpumask *cpus, + bool ordered, bool highpri) +{ + struct workqueue_attrs *attrs = alloc_workqueue_attrs(); + int ret; + + if (!attrs) + return -ENOMEM; + cpumask_copy(attrs->cpumask, cpus); + attrs->nice = highpri ? MIN_NICE : 0; + attrs->ordered = ordered; + ret = apply_workqueue_attrs(wq, attrs); + free_workqueue_attrs(attrs); + return ret; +} diff --git a/rust/kernel/cpumask.rs b/rust/kernel/cpumask.rs index 44bb36636ee32..3b82790eb0340 100644 --- a/rust/kernel/cpumask.rs +++ b/rust/kernel/cpumask.rs @@ -77,6 +77,14 @@ impl Cpumask { unsafe { &*ptr.cast() } } + /// The CPUs of `cpu`'s cluster as the architecture's topology describes it: `cpu` alone when + /// there is no cluster information. + pub fn cluster_of(cpu: CpuId) -> &'static Self { + // SAFETY: The topology keeps one mask per possible CPU for the kernel's lifetime, and `cpu` + // is a valid CPU ID by its type invariant. + unsafe { Self::from_raw(bindings::topology_cluster_cpumask(cpu.as_u32() as i32)) } + } + /// Obtain the raw `struct cpumask` pointer. pub fn as_raw(&self) -> *mut bindings::cpumask { let this: *const Self = self; diff --git a/rust/kernel/drm/gem/shmem.rs b/rust/kernel/drm/gem/shmem.rs index 83eca852139be..095f798e77d31 100644 --- a/rust/kernel/drm/gem/shmem.rs +++ b/rust/kernel/drm/gem/shmem.rs @@ -604,8 +604,19 @@ impl driver::AllocImpl for Object { prime_handle_to_fd: None, prime_fd_to_handle: None, gem_prime_import: None, - gem_prime_import_sg_table: Some(bindings::drm_gem_shmem_prime_import_sg_table), - dumb_create: Some(bindings::drm_gem_shmem_dumb_create), + // The core creates imported and dumb objects through `gem_create_object`. Without it they + // would be bare `drm_gem_shmem_object`s, which every handle lookup still treats as an + // `Object`: offer them only to drivers that can allocate the whole object. + gem_prime_import_sg_table: if T::HAS_CORE_OBJECT_ARGS { + Some(bindings::drm_gem_shmem_prime_import_sg_table) + } else { + None + }, + dumb_create: if T::HAS_CORE_OBJECT_ARGS { + Some(bindings::drm_gem_shmem_dumb_create) + } else { + None + }, dumb_map_offset: None, }; } diff --git a/rust/kernel/drm/sched.rs b/rust/kernel/drm/sched.rs index 7250c61eccd38..8194c22003c1f 100644 --- a/rust/kernel/drm/sched.rs +++ b/rust/kernel/drm/sched.rs @@ -471,6 +471,12 @@ pub struct SchedulerInner { // SchedulerInner instead, so outstanding work cannot prevent finalization. struct SchedulerOwner { allocation: Arc>, + /// The driver's submission queue, if it supplied one. Fields drop after `Drop::drop`, so the + /// queue is destroyed only after `drm_sched_fini` has stopped the scheduler's work on it. + _submit_wq: Option, + /// The driver's timeout queue, if it supplied one; possibly shared with other schedulers. This + /// reference keeps it alive until `drm_sched_fini` has synchronized the timeout work. + _timeout_wq: Option>, } impl Drop for SchedulerOwner { @@ -527,6 +533,46 @@ impl Scheduler { hang_limit, timeout_ms, name, + None, + None, + ) + } + + /// Creates a new DRM Scheduler object whose run and free work goes to `submit_wq` and whose + /// timeout work goes to `timeout_wq`, each where given. + /// + /// Without `submit_wq`, drm_sched allocates an ordered reclaim-safe queue itself. With one, + /// the scheduler owns it and destroys it after it has been finalized; like drm_sched's own it + /// must be ordered and usable for memory reclaim (see [`OwnedQueue::new_ordered_reclaim`]), so + /// that one scheduler's jobs run and are freed one at a time. + /// + /// Without `timeout_wq`, the timeout work runs on `system_percpu_wq`, which memory reclaim + /// cannot rely on. A driver whose timeout handler signals job fences passes a queue usable for + /// memory reclaim (such as [`OwnedQueue::new_percpu_reclaim`]); several schedulers may share + /// it, and each keeps a reference until it has been finalized. + /// + /// [`OwnedQueue::new_ordered_reclaim`]: crate::workqueue::OwnedQueue::new_ordered_reclaim + /// [`OwnedQueue::new_percpu_reclaim`]: crate::workqueue::OwnedQueue::new_percpu_reclaim + pub fn new_with_queues( + device: &device::Device, + num_rqs: u32, + credit_limit: u32, + hang_limit: u32, + timeout_ms: time::Msecs, + name: &'static CStr, + submit_wq: Option, + timeout_wq: Option>, + ) -> Result> { + Self::init( + UniqueArc::new_uninit(GFP_KERNEL)?, + device, + num_rqs, + credit_limit, + hang_limit, + timeout_ms, + name, + submit_wq, + timeout_wq, ) } @@ -539,6 +585,8 @@ impl Scheduler { hang_limit: u32, timeout_ms: time::Msecs, name: &'static CStr, + submit_wq: Option, + timeout_wq: Option>, ) -> Result> { // Allocate the owner before starting C workers: no fallible allocation // may abandon an initialized scheduler without running drm_sched_fini. @@ -556,8 +604,8 @@ impl Scheduler { let init_ops = bindings::drm_sched_init_args { ops: &Self::OPS, - submit_wq: core::ptr::null_mut(), - timeout_wq: core::ptr::null_mut(), + submit_wq: submit_wq.as_ref().map_or(core::ptr::null_mut(), |queue| queue.as_raw()), + timeout_wq: timeout_wq.as_ref().map_or(core::ptr::null_mut(), |queue| queue.as_raw()), num_rqs, credit_limit, hang_limit, @@ -579,7 +627,15 @@ impl Scheduler { // SAFETY: C initialized sched and PhantomData has no stored state. // No fallible operation follows successful initialization. let allocation = unsafe { sched.assume_init() }.into(); - Ok(Scheduler(owner.write(SchedulerOwner { allocation }).into())) + Ok(Scheduler( + owner + .write(SchedulerOwner { + allocation, + _submit_wq: submit_wq, + _timeout_wq: timeout_wq, + }) + .into(), + )) } } @@ -684,6 +740,8 @@ mod tests { 0, TIMEOUT_MS, c"rust-drm-sched-kunit", + None, + None, ) } diff --git a/rust/kernel/workqueue.rs b/rust/kernel/workqueue.rs index 3124062c3cad9..3e4d97583073a 100644 --- a/rust/kernel/workqueue.rs +++ b/rust/kernel/workqueue.rs @@ -298,6 +298,33 @@ impl Queue { } } + /// Returns the raw `struct workqueue_struct` pointer, for C interfaces that take a queue. + pub fn as_raw(&self) -> *mut bindings::workqueue_struct { + self.0.get() + } + + /// Enqueues a work item on `cpu`. + /// + /// This may fail if the work item is already enqueued in a workqueue. + /// + /// On a per-CPU queue the work item runs on `cpu`. On an unbound queue `cpu` only selects the + /// affinity pod (for example the CPU's last-level cache) whose worker runs it, so the + /// scheduler may pick another CPU of that pod. + pub fn enqueue_on(&self, cpu: crate::cpu::CpuId, w: W) -> W::EnqueueOutput + where + W: RawWorkItem + Send + 'static, + { + let queue_ptr = self.0.get(); + + // SAFETY: As for `enqueue`. `cpu` is a valid CPU ID by its type invariant, which is what + // `queue_work_on` requires in place of `WORK_CPU_UNBOUND`. + unsafe { + w.__enqueue(move |work_ptr| { + bindings::queue_work_on(cpu.as_u32() as ffi::c_int, queue_ptr, work_ptr) + }) + } + } + /// Enqueues a delayed work item. /// /// This may fail if the work item is already enqueued in a workqueue. @@ -374,17 +401,29 @@ impl WorkItem for ClosureWork { /// A work queue owned by the caller. /// -/// Wraps a `struct workqueue_struct` allocated with `alloc_ordered_workqueue()`. It dereferences -/// to a [`Queue`], so work items are enqueued on it exactly like on the system queues. Dropping it -/// calls `destroy_workqueue()`, which waits for every pending work item to finish first. +/// Wraps a `struct workqueue_struct` allocated with `alloc_ordered_workqueue()` or, for an +/// unbound or per-CPU queue, `alloc_workqueue()`. It dereferences to a [`Queue`], so work items +/// are enqueued on it exactly like on the system queues. Dropping it calls `destroy_workqueue()`, +/// which waits for every pending work item to finish first. +/// +/// An ordered owned queue gives a driver two things the system queues do not: its work items run +/// one at a time in queueing order, and the driver can wait for all of them at once (see +/// [`drain`]) before it tears down the state they use. An unbound owned queue runs its work items +/// in the affinity pod of the CPU they are queued for (see [`Queue::enqueue_on`]), and either kind +/// can be restricted to a set of CPUs (see [`restrict`]). A per-CPU owned queue runs each work item +/// on the CPU it is queued on, as the per-CPU system queues do. /// -/// An owned queue gives a driver two things the system queues do not: its work items run one at a -/// time in queueing order, and the driver can wait for all of them at once (see [`drain`]) before -/// it tears down the state they use. +/// A queue whose work may signal a DMA fence, or that memory reclaim otherwise waits on, must be +/// allocated by a `_reclaim` constructor (`WQ_MEM_RECLAIM`): its rescuer thread runs that work +/// when no new worker can be created because memory is short. /// /// # Invariants /// -/// The pointer is valid and points at a workqueue allocated by `alloc_ordered_workqueue()`. +/// `queue` is valid and points at a workqueue allocated by `alloc_ordered_workqueue()` when +/// `ordered` is set, or else by `alloc_workqueue()`, with `WQ_UNBOUND` when `unbound` is set +/// (`WQ_PERCPU` otherwise) and `WQ_HIGHPRI` when `highpri` is set. Ordered queues are unbound. +/// +/// [`restrict`]: OwnedQueue::restrict /// /// # Examples /// @@ -422,7 +461,12 @@ impl WorkItem for ClosureWork { /// ``` /// /// [`drain`]: OwnedQueue::drain -pub struct OwnedQueue(NonNull); +pub struct OwnedQueue { + queue: NonNull, + ordered: bool, + unbound: bool, + highpri: bool, +} impl OwnedQueue { /// Allocates an ordered workqueue named `name`. @@ -431,11 +475,78 @@ impl OwnedQueue { /// they were queued, by an unbound worker. The name is copied by the kernel and appears in the /// worker's thread name. pub fn new_ordered(name: &CStr) -> Result { + Self::ordered(name, 0) + } + + /// Allocates an ordered workqueue named `name` that may be used on the memory reclaim path + /// (`WQ_MEM_RECLAIM`), such as a GPU scheduler's submission queue. + pub fn new_ordered_reclaim(name: &CStr) -> Result { + Self::ordered(name, bindings::wq_flags_WQ_MEM_RECLAIM) + } + + fn ordered(name: &CStr, flags: u32) -> Result { // SAFETY: `name` is a valid NUL-terminated string; the C side copies it. - let ptr = unsafe { bindings::alloc_ordered_workqueue(name.as_char_ptr(), 0) }; + let ptr = unsafe { bindings::alloc_ordered_workqueue(name.as_char_ptr(), flags) }; // INVARIANT: `alloc_ordered_workqueue()` returns a valid workqueue or NULL. - Ok(Self(NonNull::new(ptr).ok_or(ENOMEM)?)) + Ok(Self { + queue: NonNull::new(ptr).ok_or(ENOMEM)?, + ordered: true, + unbound: true, + highpri: false, + }) + } + + /// Allocates an unbound workqueue named `name`, with high-priority workers if `highpri`. + /// + /// Its work items run concurrently, each in the affinity pod of the CPU it is queued for. + pub fn new_unbound(name: &CStr, highpri: bool) -> Result { + Self::flags(name, true, highpri, 0) + } + + /// Allocates an unbound workqueue named `name` that may be used on the memory reclaim path + /// (`WQ_MEM_RECLAIM`), with high-priority workers if `highpri`. + pub fn new_unbound_reclaim(name: &CStr, highpri: bool) -> Result { + Self::flags(name, true, highpri, bindings::wq_flags_WQ_MEM_RECLAIM) + } + + /// Allocates a per-CPU workqueue named `name` that may be used on the memory reclaim path + /// (`WQ_MEM_RECLAIM`), with high-priority workers if `highpri`: a driver's own counterpart of + /// `system_highpri_wq` (or `system_percpu_wq`) for work that memory reclaim may wait on. + pub fn new_percpu_reclaim(name: &CStr, highpri: bool) -> Result { + Self::flags(name, false, highpri, bindings::wq_flags_WQ_MEM_RECLAIM) + } + + fn flags(name: &CStr, unbound: bool, highpri: bool, flags: u32) -> Result { + let flags = flags + | if unbound { bindings::wq_flags_WQ_UNBOUND } else { bindings::wq_flags_WQ_PERCPU } + | if highpri { bindings::wq_flags_WQ_HIGHPRI } else { 0 }; + // SAFETY: `name` is a valid NUL-terminated string; the C side copies it. + let ptr = unsafe { bindings::alloc_workqueue_flags(name.as_char_ptr(), flags) }; + + // INVARIANT: `alloc_workqueue()` returns a valid workqueue with these flags, or NULL. + Ok(Self { + queue: NonNull::new(ptr).ok_or(ENOMEM)?, + ordered: false, + unbound, + highpri, + }) + } + + /// Restricts the queue's workers to the CPUs in `cpus`. + /// + /// An ordered queue stays ordered and a high-priority one keeps its priority. Allocates, so it + /// must be called from a context that may sleep. A per-CPU queue cannot be restricted + /// (`EINVAL`). + pub fn restrict(&self, cpus: &crate::cpumask::Cpumask) -> Result { + if !self.unbound { + return Err(EINVAL); + } + // SAFETY: The queue is a valid unbound workqueue by the type invariant and `cpus` is a + // valid mask; the helper copies it. + crate::error::to_result(unsafe { + bindings::workqueue_restrict(self.queue.as_ptr(), cpus.as_raw(), self.ordered, self.highpri) + }) } /// Waits until the workqueue is empty. @@ -446,7 +557,7 @@ impl OwnedQueue { /// stop every other producer before draining. Must be called from a context that may sleep. pub fn drain(&self) { // SAFETY: The pointer is valid by the type invariant. - unsafe { bindings::drain_workqueue(self.0.as_ptr()) }; + unsafe { bindings::drain_workqueue(self.queue.as_ptr()) }; } } @@ -456,7 +567,7 @@ impl Deref for OwnedQueue { fn deref(&self) -> &Queue { // SAFETY: The pointer is valid by the type invariant and stays valid for as long as `self` // does, which bounds the returned reference. - unsafe { Queue::from_raw(self.0.as_ptr()) } + unsafe { Queue::from_raw(self.queue.as_ptr()) } } } @@ -464,7 +575,7 @@ impl Drop for OwnedQueue { fn drop(&mut self) { // SAFETY: The pointer is valid by the type invariant. `destroy_workqueue()` drains the // queue before freeing it, so no work item can run on a freed queue. - unsafe { bindings::destroy_workqueue(self.0.as_ptr()) }; + unsafe { bindings::destroy_workqueue(self.queue.as_ptr()) }; } } diff --git a/tools/asahi/apple-dart-fw-tests.py b/tools/asahi/apple-dart-fw-tests.py index d37ff84ba29ef..7abd6c161da50 100644 --- a/tools/asahi/apple-dart-fw-tests.py +++ b/tools/asahi/apple-dart-fw-tests.py @@ -4,8 +4,8 @@ from pathlib import Path import argparse, hashlib, json, re, subprocess ROOT=Path(__file__).resolve().parents[2] -NAMES=('apple_dart_fw_handoff_enabled','apple_dart_pte_to_paddr','apple_dart_fw_table_ram','apple_dart_fw_table_decode','apple_dart_fw_slot_owned','apple_dart_fw_leaf_decode','apple_dart_fw_mapping_matches','apple_dart_fw_page_matches','apple_dart_fw_slot_mirrors','apple_dart_publish_root','apple_dart_retire_root','apple_dart_free_fw_root','apple_dart_snapshot_fw_root','apple_dart_fw_slot_foreign','apple_dart_domain_uses_fw','apple_dart_check_fw_map','apple_dart_inherited_phys','apple_dart_fw_slot_range','apple_dart_get_fw_resv_regions','apple_dart_fw_invalidate','apple_dart_hw_unmap_locked_ttbr','apple_dart_retire_error','apple_dart_t8110_hw_invalidate_tlb','apple_dart_release_device','apple_dart_domain_free') -CASES=("legacy-map-match","legacy-map-mismatch","legacy-lookup","legacy-snapshot",'mirror','changed-leaf','wrong-permission','wrong-physical','malformed-leaf','vacant','changed-owner','retire','reserved-ram','unreserved','mmio','misalignment','address-width','root-encoding','leaf-as-table','snapshot','snapshot-malformed','snapshot-unreserved','snapshot-root-change','snapshot-map-failure','snapshot-allocation-failure','range-three','range-offset','range-width','range-overflow','range-underflow','range-four','range-four-last','map-firmware','map-hole','map-host','map-owned','map-unmatched','map-attached','map-wrong-physical','map-wrong-permission','map-changed-root','map-four','lookup-inherited','lookup-changed-leaf','lookup-changed-root','lookup-unmatched','reserve-firmware','reserve-teardown','reserve-unmatched','reserve-owned','invalidate-three','invalidate-four','invalidate-changed-root','invalidate-error','core-inherited','core-hole','core-changed-leaf','core-second-dart','snapshot-unmatched','snapshot-four','range-four-width','range-three-last','handoff-matched','handoff-missing','handoff-duplicate','handoff-zero','handoff-unlocked','handoff-other-soc','retire-free','retire-free-timeout','retire-release','retire-release-timeout','retire-rollback-timeout','retire-repeat-timeout') +NAMES=('apple_dart_fw_handoff_enabled','apple_dart_pte_to_paddr','apple_dart_fw_table_page','apple_dart_fw_table_ram','apple_dart_fw_table_pte','apple_dart_fw_table_decode','apple_dart_fw_tree_table','apple_dart_fw_slot_owned','apple_dart_fw_leaf_decode','apple_dart_fw_mapping_matches','apple_dart_fw_page_matches','apple_dart_fw_slot_mirrors','apple_dart_publish_root','apple_dart_retire_root','apple_dart_free_fw_root','apple_dart_snapshot_fw_root','apple_dart_free_fw_tree','apple_dart_fw_tree_phys','apple_dart_fw_slot_foreign','apple_dart_domain_uses_fw','apple_dart_check_fw_map','apple_dart_inherited_phys_prot','apple_dart_inherited_phys','apple_dart_fw_slot_range','apple_dart_get_fw_resv_regions','apple_dart_fw_invalidate','apple_dart_hw_unmap_locked_ttbr','apple_dart_retire_error','apple_dart_t8110_hw_invalidate_tlb','apple_dart_release_device','apple_dart_domain_free') +CASES=("legacy-map-match","legacy-map-mismatch","legacy-lookup","legacy-snapshot",'mirror','changed-leaf','wrong-permission','wrong-physical','malformed-leaf','vacant','changed-owner','retire','reserved-ram','unreserved','mmio','misalignment','address-width','root-encoding','leaf-as-table','snapshot','snapshot-malformed','snapshot-unreserved','snapshot-root-change','snapshot-map-failure','snapshot-allocation-failure','range-three','range-offset','range-width','range-overflow','range-underflow','range-four','range-four-last','map-firmware','map-hole','map-host','map-owned','map-unmatched','map-attached','map-wrong-physical','map-wrong-permission','map-changed-root','map-four','lookup-inherited','lookup-changed-leaf','lookup-changed-root','lookup-unmatched','reserve-firmware','reserve-teardown','reserve-unmatched','reserve-owned','invalidate-three','invalidate-four','invalidate-changed-root','invalidate-error','core-inherited','core-hole','core-changed-leaf','core-second-dart','snapshot-unmatched','snapshot-four','range-four-width','range-three-last','handoff-matched','handoff-missing','handoff-duplicate','handoff-zero','handoff-unlocked','handoff-other-soc','retire-free','retire-free-timeout','retire-release','retire-release-timeout','retire-rollback-timeout','retire-repeat-timeout','tree-lookup','tree-changed-root','tree-changed-mid','tree-unrecorded','tree-reserve','tree-mirror-only','subtree-free-partial','subtree-table-reserved','subtree-table-unclaimed','subtree-table-claimed','subtree-table-ram','subtree-table-misalignment') def function(s,name): match=re.search(r'^static[^;{}]*\b'+re.escape(name)+r'\([^;{}]*\)\s*\{',s,re.M) if not match:return None diff --git a/tools/asahi/apple-dart-fw/fixture.c b/tools/asahi/apple-dart-fw/fixture.c index 13e39fed942d5..6c141591b63d0 100644 --- a/tools/asahi/apple-dart-fw/fixture.c +++ b/tools/asahi/apple-dart-fw/fixture.c @@ -50,7 +50,7 @@ static unsigned lock_depth; static size_t find_next_bit(unsigned long*p,size_t n,size_t a){for(;alocked_ttbr[0][0]=NULL;retiring_dart->locked_owned[0][0]=NULL;free(retired_table);retired_table=NULL;}struct iommu_resv_region*r=®ions[region_count++];r->start=s;r->length=l;r->type=type;return r;} static void list_add_tail(struct list_head*l,struct list_head*h){h->count++;} struct apple_dart_fw_root {u64*entry;u64*accepted;u64**leaf;}; +struct apple_dart_fw_tree {u64*entry;u64**mid;u64**mid_entry;u64***leaf;}; #define DART_TCR(d,s) (s) static unsigned apple_dart_readl(struct apple_dart*d,unsigned reg){return d->tcr;} static u64 firmware_leaf[2048], ours_leaf[2048], root[2048]; @@ -80,6 +81,11 @@ static int ram_valid=1,reserved=1,map_failure,allocation_failure,change_root,map static int pfn_valid(unsigned long pfn){return ram_valid&&pfn>=4&&pfn<8;} static unsigned long pfn_to_page(unsigned long pfn){return pfn;} #define PageReserved(p) (reserved) +#define IORESOURCE_MEM 0x200 +#define IORES_DESC_NONE 0 +enum {REGION_DISJOINT,REGION_INTERSECTS,REGION_MIXED}; +static int claimed; +static int region_intersects(u64 s,size_t n,unsigned long f,unsigned long d){assert(n==SZ_16K&&f==IORESOURCE_MEM&&d==IORES_DESC_NONE);return claimed?REGION_INTERSECTS:REGION_DISJOINT;} static void*allocation(size_t n){if(allocation_failure)return NULL;return calloc(1,n);} #define kzalloc(n,g) allocation(n) #define kcalloc(n,z,g) allocation((n)*(z)) @@ -207,6 +213,22 @@ int main(int argc,char**argv){ else if(!strcmp(c,"lookup-unmatched")){d.fw_handoff=0;assert(!apple_dart_inherited_phys(&domain,0));} else assert(!strcmp(c,"lookup-inherited")); } + else if(!strncmp(c,"tree-",5)){ + /* fw_tree: firmware's recorded subtree of a four-level root (slot 0, mid 0) resolves; a changed root or mid + * entry, an unrecorded tree, and a fw_mirror DART without fw_tree resolve and reserve nothing. */ + static u64 mid_table[2048];u64*entry=calloc(2048,sizeof(u64)),**mid=calloc(2048,sizeof(u64*)),**mid_entry=calloc(2048,sizeof(u64*));u64***leaves3=calloc(2048,sizeof(u64**)); + assert(entry&&mid&&mid_entry&&leaves3);mid_entry[0]=calloc(2048,sizeof(u64));leaves3[0]=calloc(2048,sizeof(u64*));assert(mid_entry[0]&&leaves3[0]); + root[0]=0x1401;entry[0]=0x1401;mid[0]=mid_table;mid_table[0]=0x1801;mid_entry[0][0]=0x1801;leaves3[0][0]=firmware_leaf; + struct apple_dart_fw_tree tree={entry,mid,mid_entry,leaves3}; + d.fw_handoff=0;d.fw_mirror=1;d.fw_tree=strcmp(c,"tree-mirror-only")!=0;d.tcr=1;d.locked_fw[0][0]=NULL; + if(strcmp(c,"tree-unrecorded")&&strcmp(c,"tree-mirror-only"))d.locked_fw_tree[0]=&tree; + if(!strcmp(c,"tree-lookup"))assert(apple_dart_inherited_phys(&domain,0)==0x8000); + else if(!strcmp(c,"tree-changed-root")){root[0]=0x1001;assert(!apple_dart_inherited_phys(&domain,0));} + else if(!strcmp(c,"tree-changed-mid")){mid_table[0]=0x1c01;assert(!apple_dart_inherited_phys(&domain,0));} + else if(!strcmp(c,"tree-unrecorded"))assert(!apple_dart_inherited_phys(&domain,0)); + else if(!strcmp(c,"tree-reserve")){apple_dart_get_fw_resv_regions(&dev,&head);assert(head.count==1&®ions[0].start==0&®ions[0].length==(1ULL<<36)&®ions[0].type==IOMMU_RESV_RESERVED);} + else {assert(!strcmp(c,"tree-mirror-only"));assert(!apple_dart_inherited_phys(&domain,0));apple_dart_get_fw_resv_regions(&dev,&head);assert(!head.count);} + } else if(!strncmp(c,"reserve-",8)){ root[0]=0x1401;d.locked_window[0]=1ULL<<40; if(!strcmp(c,"reserve-teardown")){retired_table=calloc(2048,sizeof(u64));assert(retired_table);retired_table[0]=retired_table[5]=0x1401;d.locked_ttbr[0][0]=retired_table;release_on_alloc=1;retiring_dart=&d;} @@ -269,6 +291,17 @@ int main(int argc,char**argv){ else if(!strcmp(c,"vacant")){ours[0]=0;ours[1]=0x801;assert(!apple_dart_publish_root(&d,live,owned,ours,&fw,2));assert(live[0]==0x1401&&live[1]==0x801&&owned[1]==0x801);} else if(!strcmp(c,"changed-owner")){live[0]=0x1801;assert(apple_dart_publish_root(&d,live,owned,ours,&fw,2)==-EBUSY);assert(!owned[0]);} else if(!strcmp(c,"retire")){owned[1]=0x801;live[1]=0xc01;apple_dart_retire_root(live,owned,2);assert(live[0]==0x1401&&live[1]==0xc01&&!owned[1]);} + else if(!strcmp(c,"subtree-free-partial")){ + /* A failed allocation leaves any of the four arrays missing: freeing must not touch them. */ + struct apple_dart_fw_tree*t=calloc(1,sizeof(*t));assert(t);t->leaf=calloc(2048,sizeof(u64**));assert(t->leaf);t->leaf[0]=calloc(2048,sizeof(u64*));assert(t->leaf[0]);t->leaf[0][0]=firmware_leaf; + apple_dart_free_fw_tree(&d,t);assert(unmap_calls==1); + } + /* fw_tree tables: reserved RAM, or memory nothing claims; never Linux's RAM or a device's registers. */ + else if(!strcmp(c,"subtree-table-reserved")){claimed=1;assert(apple_dart_fw_tree_table(&d,0x14000));} + else if(!strcmp(c,"subtree-table-unclaimed")){ram_valid=0;assert(apple_dart_fw_tree_table(&d,0x14000));} + else if(!strcmp(c,"subtree-table-claimed")){ram_valid=0;claimed=1;assert(!apple_dart_fw_tree_table(&d,0x14000));} + else if(!strcmp(c,"subtree-table-ram")){reserved=0;claimed=1;assert(!apple_dart_fw_tree_table(&d,0x14000));} + else if(!strcmp(c,"subtree-table-misalignment")){ram_valid=0;assert(!apple_dart_fw_tree_table(&d,0x14001));} else if(!strcmp(c,"reserved-ram")){assert(apple_dart_fw_table_ram(&d,0x14000));} else if(!strcmp(c,"unreserved")){reserved=0;assert(!apple_dart_fw_table_ram(&d,0x14000));} else if(!strcmp(c,"mmio")){ram_valid=0;assert(!apple_dart_fw_table_ram(&d,0x14000));} diff --git a/tools/asahi/g15-probe-init-tests.py b/tools/asahi/g15-probe-init-tests.py index e7617fba021c2..7ded29db0c3c7 100644 --- a/tools/asahi/g15-probe-init-tests.py +++ b/tools/asahi/g15-probe-init-tests.py @@ -78,6 +78,7 @@ def item(text, marker): static COMPLETED_REGISTRATION: Cell = const { Cell::new(0) }; static ACTIVATED: Cell = const { Cell::new(false) }; static BINDING_READY: Cell = const { Cell::new(false) }; + static QUEUE_SHARE_READY: Cell = const { Cell::new(false) }; static ATTACH_ERROR: Cell = const { Cell::new(false) }; static PROBE_ERROR: Cell = const { Cell::new(0) }; } @@ -146,6 +147,10 @@ def item(text, marker): mod binding { pub unsafe fn init() { crate::BINDING_READY.set(true); } } +// The queue-share registry must also be ready before an adapter can probe. +mod queue_share { + pub unsafe fn init() { crate::QUEUE_SHARE_READY.set(true); } +} mod cleanup { pub struct QueueOwner; impl QueueOwner { pub fn new() -> crate::Result { Ok(Self) } } @@ -157,6 +162,7 @@ def item(text, marker): impl Registration { pub fn new(name: &'static std::ffi::CStr, _module: &'static ThisModule) -> impl PinInit { assert!(BINDING_READY.get(), "adapter registered before binding registry init"); + assert!(QUEUE_SHARE_READY.get(), "adapter registered before queue-share registry init"); try_pin_init!(Self { stage: { let stage = match name.to_bytes() { @@ -208,6 +214,7 @@ def item(text, marker): fn reset(fail_alloc: bool, touch: bool, error: i32) { MODULE_READY.store(true, Ordering::Release); BINDING_READY.set(false); + QUEUE_SHARE_READY.set(false); FAIL_ALLOC.set(fail_alloc); TOUCH.set(touch); PROBE_ERROR.set(error); RETAINED.set(false); ALLOCS.set(0); PROBES.set(0); } diff --git a/tools/asahi/g17-admission-deadline-tests.py b/tools/asahi/g17-admission-deadline-tests.py index d6e63e08d1188..501c903b59f8f 100644 --- a/tools/asahi/g17-admission-deadline-tests.py +++ b/tools/asahi/g17-admission-deadline-tests.py @@ -37,15 +37,17 @@ def fixture(): mod g17 {pub fn now_ns()->u64 {crate::NOW.load(crate::Ordering::SeqCst)}} mod kernel {pub mod time {pub fn msecs_to_jiffies(ms:u32)->u64 {u64::from(ms)}}} mod hazard {pub const WAIT_MS:u32=1000;} +mod cfg {pub const RENDER_SLOTS:usize=2;} const QUARANTINE_WAIT_MS:u32=1000; #[derive(Clone,Copy,PartialEq)] enum Plan {Any,Conflict} #[derive(Clone,Copy)] struct Request<'a>(&'a ()); -struct State {owners:[u64;2],quarantined:u64,spared:u64,next_ticket:u64,waiters:usize,conflict:bool,ready:bool} +struct State {owners:[u64;cfg::RENDER_SLOTS],quarantined:u64,spared:u64,next_ticket:u64,waiters:usize,conflict:bool,ready:bool} impl State { fn exhausted(&self)->bool {false} +''' + item(source, 'fn owner_quarantine(') + r''' fn refusal(&self,_:u64)->Result {Ok(())} fn plan(&self,_:u64,_:Option>,_:Option)->Plan {if self.conflict {Plan::Conflict}else{Plan::Any}} - fn pick(&mut self,_:u64,_:Plan)->Option {if self.ready{Some(1)}else{None}} + fn pick(&mut self,_:u64,_:usize,_:Plan)->Option {if self.ready{Some(1)}else{None}} fn record(&mut self,_:u64,slot:Option,_:u64,_:Request<'_>)->Option {if slot.is_none(){self.waiters+=1;}Some(0)} fn forget_waiter(&mut self,t:Option){if t.is_some(){self.waiters-=1;}} } @@ -76,31 +78,31 @@ def fixture(): for mode in 0..2 { for conflict in [false,true] { reset(mode);let pool=new_pool(conflict,0); - let result=pool.acquire_render(1,Some(Request(&())),2_000_000_000,||Ok(())); + let result=pool.acquire_render(1,1,Some(Request(&())),2_000_000_000,||Ok(())); assert!(matches!(result,Err(EBUSY)),"busy wait must expire"); assert!(NOW.load(Ordering::SeqCst)<=2_010_000_000,"absolute deadline extended by wakes"); assert_eq!(pool.state.lock().waiters,0,"expired waiter must be removed"); reset(mode);let pool=new_pool(conflict,0); - let result=pool.acquire_render(1,Some(Request(&())),2_000_000_000,||if g17::now_ns()>=5_000_000 {Err(ECANCELED)}else{Ok(())}); + let result=pool.acquire_render(1,1,Some(Request(&())),2_000_000_000,||if g17::now_ns()>=5_000_000 {Err(ECANCELED)}else{Ok(())}); assert!(matches!(result,Err(ECANCELED)),"cancel must interrupt a busy wait"); assert!(NOW.load(Ordering::SeqCst)<=15_000_000,"cancel observation bound"); assert_eq!(pool.state.lock().waiters,0,"cancelled waiter must be removed"); } reset(mode);let pool=new_pool(false,2); - assert!(matches!(pool.acquire_render(1,None,2_000_000_000,||Ok(())),Err(EIO))); + assert!(matches!(pool.acquire_render(1,1,None,2_000_000_000,||Ok(())),Err(EIO))); assert!(NOW.load(Ordering::SeqCst)<=1_010_000_000,"quarantine budget extended"); reset(mode);let pool=new_pool(false,2);pool.state.lock().spared=2; - assert!(matches!(pool.acquire_render(1,None,2_000_000_000,||Ok(())),Err(EBUSY))); + assert!(matches!(pool.acquire_render(1,1,None,2_000_000_000,||Ok(())),Err(EBUSY))); assert!((2_000_000_000..=2_010_000_000).contains(&NOW.load(Ordering::SeqCst)),"a spared quarantine waits for the caller's deadline"); } reset(2);let pool=new_pool(true,0); - assert!(matches!(pool.acquire_render(1,Some(Request(&())),2_000_000_000,||Ok(())),Err(ERESTARTSYS))); + assert!(matches!(pool.acquire_render(1,1,Some(Request(&())),2_000_000_000,||Ok(())),Err(ERESTARTSYS))); assert_eq!(pool.state.lock().waiters,0); // A previous outer retry already spent the budget. Even available capacity // cannot restart it or install a lease after cancellation. reset(0);NOW.store(2_000_000_000,Ordering::SeqCst);let pool=new_pool(false,0);pool.state.lock().ready=true; - assert!(matches!(pool.acquire_render(1,None,2_000_000_000,||Ok(())),Err(EBUSY))); - assert!(matches!(pool.acquire_render(1,None,u64::MAX,||Err(ECANCELED)),Err(ECANCELED))); + assert!(matches!(pool.acquire_render(1,1,None,2_000_000_000,||Ok(())),Err(EBUSY))); + assert!(matches!(pool.acquire_render(1,1,None,u64::MAX,||Err(ECANCELED)),Err(ECANCELED))); assert_eq!(WAITS.load(Ordering::SeqCst),0); println!("PASS production admission: all-pairs-busy/no-progress and 20,000 unrelated wakes expire at absolute deadline; no-event cancellation bounded; conflict/quarantine/signal cleanup preserved"); } diff --git a/tools/asahi/g17-admission-order-tests.py b/tools/asahi/g17-admission-order-tests.py index e247afd638318..849fcefcbe576 100644 --- a/tools/asahi/g17-admission-order-tests.py +++ b/tools/asahi/g17-admission-order-tests.py @@ -29,49 +29,86 @@ def main(): #![allow(dead_code)] use std::sync::atomic::{AtomicUsize,AtomicBool,Ordering}; type Result = std::result::Result; -const ECANCELED:i32=-125;const EIO:i32=-5;const EFAULT:i32=-14; -mod g17 {pub mod fw {pub mod compute_pool { - pub fn required_blocks(_:usize,cores:u32,pages:u16)->crate::Result { - if cores==0 {Err(-22)} else {Ok(21+usize::from(pages))} +const ECANCELED:i32=-125;const EIO:i32=-5;const EFAULT:i32=-14;const EINVAL:i32=-22; +mod g17 { +pub mod fw { + pub mod compute_pool { + pub fn required_blocks(_:usize,cores:u32,pages:u16)->crate::Result { + if cores==0 {Err(-22)} else {Ok(21+usize::from(pages))} + } } -}}} -struct Status(i32); -impl Status {fn get(&self)->i32{self.0}} -struct Context {status:Status} -impl Context {fn status(&self)->&Status{&self.status} fn is_current(&self)->bool{true}} -struct Shared {compute_blocks:usize,active_cores:u32,crashed:AtomicBool} -struct Backend {shared:Shared,physical:AtomicUsize,closed:AtomicBool,context:Context} -impl Backend { - fn ensure_compute(&self,_:u16)->Result {self.physical.fetch_add(1,Ordering::SeqCst);Ok(())} -''' + method('runtime.rs', 'fn admission_open(') + r''' + pub mod m5 { pub mod pool { #[derive(Clone,Copy,Debug,PartialEq)] pub struct PoolGeometry(pub u32); } } } -trait Api {fn ensure_compute(&self,scs_pages:u16)->Result;} -impl Api for Backend { +pub mod freelist { + #[derive(Debug,PartialEq)] + pub enum ComputeCapacity { Blocks(usize), M5(Option) } +} +pub mod render_resources { #[derive(Clone,Copy)] pub struct Plan { pub geometry: super::fw::m5::pool::PoolGeometry } } +pub mod command { + pub struct ScratchRequest(pub u16); + impl ScratchRequest { pub fn pages(&self)->u16{self.0} } + pub enum Validated { + Render, + Compute { plan: Option, scratch: ScratchRequest }, + } +} +pub mod runtime { + use super::*; + use crate::{Result,ECANCELED,EIO,EFAULT,EINVAL}; + use std::sync::atomic::{AtomicUsize,AtomicBool,Ordering}; + pub struct Status(pub i32); + impl Status {fn get(&self)->i32{self.0}} + pub struct Context {pub status:Status,pub m5:bool} + impl Context {fn status(&self)->&Status{&self.status} fn is_current(&self)->bool{true} fn is_m5(&self)->bool{self.m5}} + pub struct Shared {pub compute_blocks:usize,pub active_cores:u32,pub crashed:AtomicBool} + pub struct Admission; + pub struct Backend {pub shared:Shared,pub physical:AtomicUsize,pub closed:AtomicBool,pub context:Context} + impl Backend { + pub fn ensure_compute(&self,_:freelist::ComputeCapacity,_:&Admission)->Result {self.physical.fetch_add(1,Ordering::SeqCst);Ok(())} +''' + method('runtime.rs', 'pub(super) fn compute_capacity(').replace('pub(super) ', 'pub ', 1) + '\n' + method('runtime.rs', 'fn admission_open(').replace('fn ', 'pub fn ', 1) + r''' + } + pub trait Api {fn ensure_compute(&self,command:&command::Validated)->Result;} + impl Api for Backend { ''' + method('runtime/backend.rs', 'fn ensure_compute(') + r''' + } +} +} +use g17::{command::{ScratchRequest,Validated},freelist::ComputeCapacity,fw::m5::pool::PoolGeometry,render_resources::Plan,runtime::*}; +fn compute(pages:u16)->Validated {Validated::Compute{plan:None,scratch:ScratchRequest(pages)}} +fn backend(cores:u32,m5:bool)->Backend { + Backend{shared:Shared{compute_blocks:21,active_cores:cores,crashed:AtomicBool::new(false)},physical:AtomicUsize::new(0),closed:AtomicBool::new(false),context:Context{status:Status(0),m5}} } fn main(){ - let backend=Backend{shared:Shared{compute_blocks:21,active_cores:10,crashed:AtomicBool::new(false)},physical:AtomicUsize::new(0),closed:AtomicBool::new(false),context:Context{status:Status(0)}}; + let backend=backend(10,false); // Four clients each enqueue nine queues in reverse dependency order. The // final host signal cannot be submitted if ioctl admission blocks on channels. - for _client in 0..4 {for _queue in (0..9).rev() {::ensure_compute(&backend,0).unwrap();}} + for _client in 0..4 {for _queue in (0..9).rev() {::ensure_compute(&backend,&compute(0)).unwrap();}} assert_eq!(backend.physical.load(Ordering::SeqCst),0,"unsignalled work must not pin physical capacity"); - Backend::ensure_compute(&backend,0).unwrap(); + assert_eq!(backend.compute_capacity(&compute(3)),Ok(ComputeCapacity::Blocks(24))); + Backend::ensure_compute(&backend,backend.compute_capacity(&compute(0)).unwrap(),&Admission).unwrap(); assert_eq!(backend.physical.load(Ordering::SeqCst),1); - let invalid=Backend{shared:Shared{compute_blocks:21,active_cores:0,crashed:AtomicBool::new(false)},physical:AtomicUsize::new(0),closed:AtomicBool::new(false),context:Context{status:Status(0)}}; - assert_eq!(::ensure_compute(&invalid,0),Err(-22)); + let invalid=self::backend(0,false); + assert_eq!(::ensure_compute(&invalid,&compute(0)),Err(-22)); assert_eq!(invalid.physical.load(Ordering::SeqCst),0); + assert_eq!(::ensure_compute(&backend,&Validated::Render),Err(EINVAL)); + // M5 sizes a compute pool by its plan's geometry; that, too, admits nothing physical. + let m5=self::backend(0,true); + let planned=Validated::Compute{plan:Some(Plan{geometry:PoolGeometry(7)}),scratch:ScratchRequest(0)}; + for _ in 0..36 {::ensure_compute(&m5,&planned).unwrap();} + assert_eq!(m5.compute_capacity(&planned),Ok(ComputeCapacity::M5(Some(PoolGeometry(7))))); + assert_eq!(m5.physical.load(Ordering::SeqCst),0,"unsignalled work must not pin physical capacity"); for _ in 0..512 { backend.closed.store(true,Ordering::Release); - assert_eq!(backend.admission_open().and_then(|()| Backend::ensure_compute(&backend,0)),Err(ECANCELED),"closed constructor must not install"); + assert_eq!(backend.admission_open().and_then(|()| Backend::ensure_compute(&backend,ComputeCapacity::Blocks(21),&Admission)),Err(ECANCELED),"closed constructor must not install"); assert_eq!(backend.physical.load(Ordering::SeqCst),1); } - println!("PASS closed builders cannot install after logical close; production ioctl sizing validates 36 reverse-order submissions without physical admission; invalid sizing consumes no channel"); + println!("PASS closed builders cannot install after logical close; production ioctl sizing validates 36 reverse-order submissions without physical admission, on G17P and M5; invalid sizing consumes no channel"); } ''' - fixed = 'crate::g17::fw::compute_pool::required_blocks(self.shared.compute_blocks, self.shared.active_cores, scs_pages)\n .map(|_| ())' + fixed = 'self.compute_capacity(command).map(|_| ())' assert code.count(fixed) == 1 with tempfile.TemporaryDirectory(prefix='g17-admission-order-') as tmp: - for name, source in [('fixed', code), ('eager-physical-admission', code.replace(fixed, 'Backend::ensure_compute(self, scs_pages)')), ('ignore-logical-close', code.replace('self.closed.load(Ordering::Acquire)', 'false'))]: + for name, source in [('fixed', code), ('eager-physical-admission', code.replace(fixed, 'self.compute_capacity(command).and_then(|required| Backend::ensure_compute(self, required, &Admission))')), ('ignore-logical-close', code.replace('self.closed.load(Ordering::Acquire)', 'false'))]: path = Path(tmp) / (name + '.rs') binary = path.with_suffix('') path.write_text(source) diff --git a/tools/asahi/g17-capacity-fault-tests.py b/tools/asahi/g17-capacity-fault-tests.py index c03764ca28d60..bd79442ef05a9 100644 --- a/tools/asahi/g17-capacity-fault-tests.py +++ b/tools/asahi/g17-capacity-fault-tests.py @@ -33,9 +33,14 @@ def fixture(): 'pub(crate) fn recycle_quarantined_idle(', 'pub(crate) fn recycle_killed(', 'fn recycle_settled(', 'fn release_witnessed_inner(', 'pub(crate) fn share_idle(', 'pub(crate) fn shareable(', 'pub(crate) fn can_rearm(', 'pub(crate) fn recycled_pending(']) exit_source = (ROOT / 'queue/compute/exit.rs').read_text() - methods += '\n' + item(exit_source, 'pub(crate) fn settle_exit(').replace('super::Previous', 'Previous').replace('crate::g17::qos::Owner', 'qos::Owner') + methods += '\n' + (item(exit_source, 'pub(crate) fn settle_exit(').replace('super::Previous', 'Previous') + .replace('crate::g17::qos::Owner', 'qos::Owner').replace('super::retirement::', 'retirement::')) drain = item((ROOT / 'mod.rs').read_text(), 'fn drain_reclaims(') proof = item(retirement, 'pub(crate) const fn stamp_newer(') + '\n' + item(retirement, 'pub(crate) const fn stamp_covers(') + '\n' + item(retirement, 'pub(crate) fn quarantined_retired(') + # M5 judges retained commands by their completion records. + proof += '\n' + '#[derive(Clone, Copy, Debug, PartialEq, Eq)]\n' + item(retirement, 'pub(crate) struct Observation') + proof += '\n' + next(line for line in retirement.splitlines() if 'const CANCELLED_TICKS:' in line) + proof += '\n' + item(retirement, 'pub(crate) fn unexecuted(') return r''' #![allow(dead_code)] use std::sync::{Arc, atomic::{AtomicUsize, AtomicU32, Ordering}}; @@ -50,6 +55,7 @@ def fixture(): struct Context { id: u64, live: Arc, killed: AtomicU32 } impl Context { fn note_compute_binding(&self, _: bool) {} + fn retain_firmware_pages(&self) {} fn new(id: u64, live: &Arc) -> Arc { live.fetch_add(1, Ordering::SeqCst); Arc::new(Self { id, live: live.clone(), killed: AtomicU32::new(1) }) @@ -64,8 +70,9 @@ def fixture(): } #[derive(Clone, Copy)] struct Cursors { count: u32, producer: u32, consumer: u32 } ''' + proof + r''' -mod retirement { pub(crate) use super::{quarantined_retired, stamp_newer}; } -struct Graph { cursors: Cursors, stamp: u32, fail: bool, resets: usize, owner: u64 } +mod retirement { pub(crate) use super::{quarantined_retired, stamp_newer, unexecuted}; } +// These queues are G17P's (m5 false); the M5 branches compile but do not run. +struct Graph { cursors: Cursors, stamp: u32, fail: bool, resets: usize, owner: u64, m5: bool } impl Graph { fn set_owner(&mut self, context: &Context) -> Result { if self.fail { Err(EIO) } else { self.owner = context.id; Ok(()) } } fn reset_items(&mut self) -> Result { if self.fail { Err(EIO) } else { self.resets += 1; Ok(()) } } @@ -75,7 +82,7 @@ def fixture(): struct Kick; impl Kick { fn clear_parent_after_recovery(&mut self) {} } #[derive(Clone, Copy, PartialEq, Eq)] -enum Phase { Published, Acknowledged, Unproven, Released } +enum Phase { Published, Acknowledged, Unproven, Witnessed, Released } struct Exit { phase: Phase } impl Exit { fn released(&self) -> bool { self.phase == Phase::Released } } struct State { killed: u32 } @@ -85,9 +92,10 @@ def fixture(): fn has_firmware_consumer(&self) -> bool { false } fn compute_words(&self) -> Result<[u32; 2]> { Ok([0, 0]) } fn replay_witness_words(&self) -> Option<[u32; 2]> { Some([0, 0]) } + fn signalled(&self) -> bool { false } } struct Packet { completion: Arc } -struct Active { packet: Arc, ticket: Option, qos_pending: bool } +struct Active { packet: Arc, ticket: Option, observed: Option, qos_pending: bool } struct Fifo { entries: Vec, len: usize } impl Fifo { fn new() -> Self { Self { entries: vec![], len: 0 } } @@ -110,6 +118,8 @@ def fixture(): retire_pending: bool, retirement_proved: bool, replay_retirement: bool, replay_error: Option, replay_prepared: bool, submitted: u32, kick: Kick, config_cache: Option<()>, source_hint: ReaderHint, source_readers: AtomicU32, claims: u32, fault_stamp: Option, + configuration: Option<()>, retained_owner: Option<()>, timed_out_at: u64, + m5_install: Option<()>, published_config: Option<()>, outer_started: u32, ordinal: u64, } impl Queue { fn valid_ticket(&self, _: &Active, _: &Ticket) -> Result { Ok(()) } @@ -121,7 +131,7 @@ def fixture(): Queue { owner: Some(2), installed: true, active: Fifo::new(), quarantined: false, previous: None, unpublished_binding: false, released: false, - graph: Graph { cursors: Cursors { count: 1280, producer: 4, consumer: 4 }, stamp: 7 << 8, fail: false, resets: 0, owner: 2 }, + graph: Graph { cursors: Cursors { count: 1280, producer: 4, consumer: 4 }, stamp: 7 << 8, fail: false, resets: 0, owner: 2, m5: false }, binding: Some(Binding { context: Context::new(2, live) }), retirement_ready: false, released_at: 0, growing: false, pool_phase: PoolPhase::Populated, needs_fresh_backing: false, exit: None, replays: Fifo::new(), @@ -130,14 +140,16 @@ def fixture(): retire_pending: false, retirement_proved: false, replay_retirement: false, replay_error: None, replay_prepared: false, submitted: 0, kick: Kick, fault_stamp: None, config_cache: None, source_hint: ReaderHint { possible: false }, source_readers: AtomicU32::new(0), claims: 0, + configuration: None, retained_owner: None, timed_out_at: 0, + m5_install: None, published_config: None, outer_started: 0, ordinal: 0, } } fn fault(live: &Arc) -> Queue { let mut queue = idle(live); let old = Context::new(1, live); - queue.previous = Some(Previous { binding: Some(Binding { context: old.clone() }), context: old, publication: Some((11, 3)) }); + queue.previous = Some(Previous { binding: Some(Binding { context: old.clone() }), context: old, publication: Some((11, 3)), moved: false }); queue.active.push(Active { packet: Arc::new(Packet { completion: Arc::new(Completion) }), - ticket: Some(Ticket { owner: 2, item_count: 1280, descriptor: 0x1000, item_producer: 4, kick: 7 }), qos_pending: true }).unwrap(); + ticket: Some(Ticket { owner: 2, item_count: 1280, descriptor: 0x1000, item_producer: 4, kick: 7 }), observed: None, qos_pending: true }).unwrap(); queue.quarantined = true; queue.awaiting_witness = true; queue.submitted = 1; @@ -259,7 +271,7 @@ def fixture(): } let mut queue = idle(&live); let old = Context::new(1, &live); - queue.previous = Some(Previous { binding: Some(Binding { context: old.clone() }), context: old, publication: None }); + queue.previous = Some(Previous { binding: Some(Binding { context: old.clone() }), context: old, publication: None, moved: false }); queue.release_owner().unwrap(); assert_eq!(queue.graph.owner, 1); assert!(queue.previous.is_none()); diff --git a/tools/asahi/g17-compute-constructor-tests.py b/tools/asahi/g17-compute-constructor-tests.py index 14b736f50436b..3c96caec2c19a 100644 --- a/tools/asahi/g17-compute-constructor-tests.py +++ b/tools/asahi/g17-compute-constructor-tests.py @@ -53,7 +53,7 @@ def fixture(): } struct Shared{preparations:Gate,state:State,changed:Changed,crashed:AtomicBool} impl Shared{fn drm(&self)->Result<()>{Ok(())} fn note_changed(&self){self.changed.notify_all();}} -struct Context;impl Context{fn vm(&self)->&(){&()}} +struct Context{m5:bool}impl Context{fn vm(&self)->&(){&()} fn is_m5(&self)->bool{self.m5}} struct Backend{shared:Shared,context:Context} struct Allocator<'a>{dev:&'a (),uat:&'a ()} struct ComputeShared;struct ComputeScheduler; @@ -76,7 +76,7 @@ def fixture(): fn main(){ for scheduler in [false,true]{for cancel in [false,true]{for building in [false,true]{ NOW.store(0,Ordering::SeqCst);CANCEL.store(cancel,Ordering::SeqCst);ALLOCATE.store(!building,Ordering::SeqCst); - let b=Backend{context:Context,shared:Shared{preparations:Gate,changed:Changed,crashed:AtomicBool::new(false),state:State(Mutex::new(Some(Box::new(Firmware{uat:(),init:Init{shared:false,scheduler:false},queues:Queues{compute_building:building,compute_scheduler_building:building}}))))}}; + let b=Backend{context:Context{m5:false},shared:Shared{preparations:Gate,changed:Changed,crashed:AtomicBool::new(false),state:State(Mutex::new(Some(Box::new(Firmware{uat:(),init:Init{shared:false,scheduler:false},queues:Queues{compute_building:building,compute_scheduler_building:building}}))))}}; let check=||{if CANCEL.load(Ordering::SeqCst)&&now_ns()>=10_000_000{Err(ECANCELED)}else{Ok(())}}; let admission=Admission::new(20_000_000,&check); let alloc=Allocator{dev:&(),uat:&()}; @@ -87,7 +87,13 @@ def fixture(): assert!(!fw.init.shared&&!fw.init.scheduler,"expired constructor must not install"); if !building{assert!(!fw.queues.compute_building&&!fw.queues.compute_scheduler_building,"failed constructor must clear builder flag");} }}} - println!("PASS 8 production shared/scheduler constructor waits: expiry, cancellation, off-lock-build expiry, no installation and flag cleanup"); + // M5 compute queues need no shared aperture: nothing waits, builds or installs. + NOW.store(0,Ordering::SeqCst);CANCEL.store(false,Ordering::SeqCst);ALLOCATE.store(true,Ordering::SeqCst); + let b=Backend{context:Context{m5:true},shared:Shared{preparations:Gate,changed:Changed,crashed:AtomicBool::new(false),state:State(Mutex::new(Some(Box::new(Firmware{uat:(),init:Init{shared:false,scheduler:false},queues:Queues{compute_building:true,compute_scheduler_building:false}}))))}}; + let check=||Ok(()); + assert_eq!(b.shared_compute(&Admission::new(20_000_000,&check)),Ok(())); + assert_eq!(now_ns(),0);assert!(!b.shared.state.lock().as_ref().unwrap().init.shared); + println!("PASS 8 production shared/scheduler constructor waits: expiry, cancellation, off-lock-build expiry, no installation and flag cleanup; M5 skips the G17P shared aperture"); } ''' diff --git a/tools/asahi/g17-compute-deadline-tests.py b/tools/asahi/g17-compute-deadline-tests.py index e33f643fc61f0..45e95c0026ead 100644 --- a/tools/asahi/g17-compute-deadline-tests.py +++ b/tools/asahi/g17-compute-deadline-tests.py @@ -33,12 +33,15 @@ def fixture(): for old, new in [('super::job::Packet', 'Packet'), ('super::command::Validated', 'Validated'), ('super::recovery::Memory', 'Memory'), - ('super::fw::compute_pool::required_blocks', 'required_blocks')]: + ('super::fw::compute_pool::required_blocks', 'required_blocks'), + ('super::freelist::ComputeCapacity', 'ComputeCapacity'), + ('super::preparation::QUEUE_WAIT_NS', 'QUEUE_WAIT_NS')]: packet = packet.replace(old, new) ensure = ensure.replace(old, new) admission = (item(preparation, "pub(super) struct Admission<'a>") + '\n' + item(preparation, "impl<'a> Admission<'a>")).replace('pub(super) ', '').replace('super::now_ns()', 'now_ns()').replace('super::timeout::', '') gate = item(preparation, 'pub(super) fn enter_until(').replace('pub(super) ', '') + queue_wait = next(line for line in preparation.splitlines() if 'const QUEUE_WAIT_NS:' in line).replace('pub(super) ', '') return r''' #![allow(dead_code,unused_variables,irrefutable_let_patterns)] use std::cell::Cell; @@ -47,11 +50,14 @@ def fixture(): const EINVAL:i32=-22;const ECANCELED:i32=-125;const EIO:i32=-5;const ENODEV:i32=-19;const EBUSY:i32=-16;const EAGAIN:i32=-11; const RECOVERY_SETTLE_POLLS:usize=200;const WORK_RING_WAIT_NS:u64=2_000_000_000; const RECOVERY_WAIT_MS:u32=26_000;const RECOVERY_WAIT_NS:u64=RECOVERY_WAIT_MS as u64*1_000_000; +''' + queue_wait + r''' +// A compute publication waits up to QUEUE_WAIT_NS, or a recovery's wait if that is longer. +const PUBLICATION_WAIT_NS:u64=if QUEUE_WAIT_NS>RECOVERY_WAIT_NS{QUEUE_WAIT_NS}else{RECOVERY_WAIT_NS}; static NOW:AtomicU64=AtomicU64::new(0);static CALLS:AtomicUsize=AtomicUsize::new(0); static MODE:AtomicUsize=AtomicUsize::new(0);static CANCEL:AtomicU64=AtomicU64::new(u64::MAX); static STATUS:AtomicU64=AtomicU64::new(u64::MAX);static CLOSED:AtomicBool=AtomicBool::new(false); fn now_ns()->u64{NOW.load(Ordering::SeqCst)} -fn advance(ns:u64){NOW.fetch_add(ns,Ordering::SeqCst);assert!(now_ns()<=RECOVERY_WAIT_NS+4_000_000_000,"admission budget exceeded");} +fn advance(ns:u64){NOW.fetch_add(ns,Ordering::SeqCst);assert!(now_ns()<=PUBLICATION_WAIT_NS+4_000_000_000,"admission budget exceeded");} mod kernel{pub mod time{ pub fn msecs_to_jiffies(ms:u32)->u32{ms} pub struct Delta(pub u64);impl Delta{pub fn from_millis(ms:u64)->Self{Self(ms*1_000_000)}pub fn from_micros(us:i64)->Self{Self(us as u64*1_000)}} @@ -98,7 +104,13 @@ def fixture(): struct ComputeDemand;impl ComputeDemand{fn new(_:&Shared)->Self{Self}} struct Backend{shared:Shared,owner:u64,context:Context} fn required_blocks(_:usize,_:u32,_:u16)->Result{Ok(1)} +type ComputeCapacity=usize; impl Backend{ + fn compute_capacity(&self,command:&Validated)->Result{let Validated::Compute{scratch}=command;required_blocks(self.shared.compute_blocks,self.shared.active_cores,scratch.pages())} + // Off M5 there is no shared work ring: no outer domain, and a claim always holds. + fn outer_domain(&self)->Option>{None} + fn claim_outer(&self,_:&Firmware,engine:g17::job::Engine)->Result{assert_eq!(engine,g17::job::Engine::Compute,"compute admission takes a compute work state");Ok(true)} + fn publish_claimed(&self,firmware:&mut Firmware,packet:&Arc,dependencies:&[()],deferred:&mut DeferredBatch)->Result{firmware.publish_compute(self.owner,packet.clone(),dependencies,deferred)} fn compute_direct(&self,_:&Arc,_:usize,_:&mut DeferredBatch)->Option{None} fn sources_published(&self,_:&Arc)->Result{Ok(true)} fn remember_direct(&self,_:&Queues){} @@ -122,9 +134,9 @@ def fixture(): MODE.store(mode,Ordering::SeqCst);NOW.store(0,Ordering::SeqCst);CALLS.store(0,Ordering::SeqCst); let r=b.compute_packet(&p,&mut DeferredBatch); assert_eq!(r,Err(EBUSY),"one admission budget covers binding theft/recovery/publication"); - let budget=if mode==1{RECOVERY_WAIT_NS}else{WORK_RING_WAIT_NS}; + let budget=if mode==1{PUBLICATION_WAIT_NS}else{QUEUE_WAIT_NS}; assert!(now_ns()>=budget&&now_ns()<=budget+10_000_000,"admission budget exceeded"); - if mode==0{assert_eq!(CALLS.load(Ordering::SeqCst),4);assert_eq!(now_ns(),WORK_RING_WAIT_NS);} + if mode==0{assert_eq!(CALLS.load(Ordering::SeqCst),(QUEUE_WAIT_NS/500_000_000) as usize);assert_eq!(now_ns(),QUEUE_WAIT_NS);} println!("PASS mode={} result={:?} calls={} elapsed_ms={}",mode,r,CALLS.load(Ordering::SeqCst),now_ns()/1_000_000); } // A context without a free work state waits for one under the admission @@ -132,7 +144,7 @@ def fixture(): MODE.store(4,Ordering::SeqCst);NOW.store(0,Ordering::SeqCst);CALLS.store(0,Ordering::SeqCst); let r=b.compute_packet(&p,&mut DeferredBatch); assert_eq!(r,Err(EBUSY),"a busy work state ends at the admission deadline"); - assert!(now_ns()>=WORK_RING_WAIT_NS&&now_ns()<=WORK_RING_WAIT_NS+10_000_000,"a busy work state waits for the admission deadline"); + assert!(now_ns()>=QUEUE_WAIT_NS&&now_ns()<=QUEUE_WAIT_NS+10_000_000,"a busy work state waits for the admission deadline"); println!("PASS busy work state waits elapsed_ms={}",now_ns()/1_000_000); for status in [false,true] { MODE.store(2,Ordering::SeqCst);NOW.store(0,Ordering::SeqCst);CALLS.store(0,Ordering::SeqCst); @@ -146,6 +158,11 @@ def fixture(): let admission=Admission::new(WORK_RING_WAIT_NS,&check); assert!(matches!(b.shared.preparations.enter_until(&admission),Err(EBUSY)),"spent budget before available gate"); assert_eq!(b.ensure_compute(0,&admission),Err(EBUSY),"spent budget before nested admission"); + // A pending drain witness extends a shorter admission to the recovery wait. + MODE.store(2,Ordering::SeqCst);NOW.store(0,Ordering::SeqCst); + let admission=Admission::new(WORK_RING_WAIT_NS,&check); + assert_eq!(b.ensure_compute(0,&admission),Err(EBUSY)); + assert!(now_ns()>=RECOVERY_WAIT_NS&&now_ns()<=RECOVERY_WAIT_NS+10_000_000,"a pending witness waits out the recovery extension"); for cancel in [false,true] { NOW.store(0,Ordering::SeqCst);CLOSED.store(true,Ordering::SeqCst); let check=||{if cancel&&now_ns()>=10_000_000{Err(ECANCELED)}else{Ok(())}}; @@ -162,14 +179,14 @@ def main(): source = fixture() mutants = [ ('fixed', source, ''), - ('renew-on-binding-loss', source.replace('let admission = Admission::new(deadline, &check);\n let progress = &self.shared.progress;\n loop {', 'let progress = &self.shared.progress;\n loop {\n let admission = Admission::new(now_ns().saturating_add(WORK_RING_WAIT_NS), &check);'), 'admission budget exceeded'), + ('renew-on-binding-loss', source.replace('let admission = Admission::new(deadline, &check);\n let progress = &self.shared.progress;\n let outer = self.outer_domain();\n loop {', 'let progress = &self.shared.progress;\n let outer = self.outer_domain();\n loop {\n let admission = Admission::new(now_ns().saturating_add(QUEUE_WAIT_NS), &check);'), 'admission budget exceeded'), ('ignore-deadline', source.replace('if now_ns() >= self.deadline.get()', 'if false'), 'admission budget exceeded'), ('render-work-state', source.replace('acquire(crate::g17::job::Engine::Compute)', 'acquire(crate::g17::job::Engine::Render)'), 'compute admission takes a compute work state'), ('omit-gate-check', source.replace(' admission.check()?;\n match self.try_enter()', ' match self.try_enter()').replace('admission.slice_ms()?', '10'), 'gate wake storm'), - ('no-recovery-extension', source.replace('admission.extend_for_recovery();', ''), 'admission budget exceeded'), + ('no-recovery-extension', source.replace('admission.extend_for_recovery();', ''), 'recovery extension'), ] # Bind the old unbounded mutant with the review's eventual cancellation. - mutants[1] = (mutants[1][0], mutants[1][1].replace('for mode in 0..4 {', 'CANCEL.store(40_000_000_000,Ordering::SeqCst);\n for mode in 0..4 {'), mutants[1][2]) + mutants[1] = (mutants[1][0], mutants[1][1].replace('for mode in 0..4 {', 'CANCEL.store(PUBLICATION_WAIT_NS+10_000_000_000,Ordering::SeqCst);\n for mode in 0..4 {'), mutants[1][2]) with tempfile.TemporaryDirectory(prefix='g17-compute-deadline-') as tmp: for name, code, expected in mutants: if name != 'fixed': diff --git a/tools/asahi/g17-compute-growth-deadline-tests.py b/tools/asahi/g17-compute-growth-deadline-tests.py index f2c148e5ec46a..4cff04e4343e0 100644 --- a/tools/asahi/g17-compute-growth-deadline-tests.py +++ b/tools/asahi/g17-compute-growth-deadline-tests.py @@ -29,7 +29,8 @@ def fixture(): admission = (item(prep, "pub(super) struct Admission<'a>") + '\n' + item(prep, "impl<'a> Admission<'a>")).replace('pub(super) ', '').replace('super::now_ns()', 'now_ns()').replace('super::timeout::', '') methods = '\n'.join(item(runtime, sig) for sig in [' fn grow_compute(', ' fn finish_uninstalled_compute(']) - methods = methods.replace('super::object::Allocator', 'Allocator').replace('super::freelist::Pages', 'Pages') + methods = (methods.replace('super::object::Allocator', 'Allocator').replace('super::freelist::Pages', 'Pages') + .replace('super::freelist::ComputeCapacity', 'ComputeCapacity')) return r''' #![allow(dead_code)] use std::cell::Cell; @@ -55,13 +56,16 @@ def fixture(): struct Available;impl Available{fn observe(&self)->u64{now_ns()}fn notify(&self){}} struct Shared{preparations:Gate,state:State,crashed:AtomicBool,progress:Available,channels:Available} impl Shared{fn drm(&self)->Result<()>{Ok(())}fn queue_events(&self){}} -struct Context;impl Context{fn vm(&self)->&(){&()}} +struct Vm;impl Vm{fn memory_client(&self)->Option<&()>{None}} +struct Context;impl Context{fn vm(&self)->&Vm{&Vm}} +// A G17P compute pool is sized in blocks; the sizes only flow through here. +type ComputeCapacity=usize; struct Backend{shared:Shared,context:Context,owner:u64} struct Allocator<'a>{dev:&'a (),uat:&'a ()} struct Pages;struct Alias; impl Pages{ - fn compute_sized(_:&Allocator<'_>,_:usize)->Result{PAGES.store(true,Ordering::SeqCst);Ok(Self)} - fn map_client(&self,_:&())->Result{ALIAS.store(true,Ordering::SeqCst);Ok(Alias)} + fn for_compute(_:&Allocator<'_>,_:ComputeCapacity,_:Option<&()>)->Result{PAGES.store(true,Ordering::SeqCst);Ok(Self)} + fn map_client(&self,_:&Vm)->Result{ALIAS.store(true,Ordering::SeqCst);Ok(Alias)} } impl Drop for Pages{fn drop(&mut self){PAGES.store(false,Ordering::SeqCst);}} impl Drop for Alias{fn drop(&mut self){ALIAS.store(false,Ordering::SeqCst);}} @@ -71,7 +75,7 @@ def fixture(): fn pool_fits(&self,_:usize)->bool{false} fn begin_growth(&mut self,_:&(),_:usize)->Result{Ok(false)} fn growth_token(&self)->Option{Some(7)} - fn finish_growth(&mut self,_:&(),_:&mut Option,_:&mut Option,_:usize)->Result{Err(EAGAIN)} + fn finish_growth(&mut self,_:&(),_:&mut Option,_:&mut Option)->Result{Err(EAGAIN)} fn abort_growth(&mut self,t:u64)->Result>{assert_eq!(t,7);ABORTED.store(true,Ordering::SeqCst);Ok(Some(self.fresh))} } impl Drop for Queue{fn drop(&mut self){if self.fresh{assert!(RESERVED.load(Ordering::SeqCst));assert!(!ALIAS.load(Ordering::SeqCst)&&!PAGES.load(Ordering::SeqCst),"growth aliases must precede fresh QID return");}}} diff --git a/tools/asahi/g17-growth-tests.py b/tools/asahi/g17-growth-tests.py index 2150f121eaa6f..3f583ea3e0302 100644 --- a/tools/asahi/g17-growth-tests.py +++ b/tools/asahi/g17-growth-tests.py @@ -23,24 +23,26 @@ def fixture(): const EIO:i32=-5;const EBUSY:i32=-16;const EINVAL:i32=-22; fn now_ns()->u64{123} struct Context {live:Arc} -impl Context {fn note_compute_binding(&self,_:bool){} fn new(live:&Arc)->Arc{live.fetch_add(1,Ordering::SeqCst);Arc::new(Self{live:live.clone()})}} +impl Context {fn note_compute_binding(&self,_:bool){} fn retain_firmware_pages(&self){} fn new(live:&Arc)->Arc{live.fetch_add(1,Ordering::SeqCst);Arc::new(Self{live:live.clone()})}} impl Drop for Context {fn drop(&mut self){self.live.fetch_sub(1,Ordering::SeqCst);}} struct Binding {context:Arc,pool_alias:Option,descriptors:Option} -struct Previous {context:Arc,binding:Option,publication:Option<(u64,u64)>} +struct Previous {context:Arc,binding:Option,publication:Option<(u64,u64)>,moved:bool} +// A G17P compute pool is sized in blocks; M5's geometry does not reach this queue. +#[derive(Clone,Copy,Debug,PartialEq)] enum ComputeCapacity{Blocks(usize)} #[derive(Clone,Copy,PartialEq)] enum PoolPhase{Populated,ReleaseSent(u32),Vacant} struct Fifo {len:usize} struct Graph; impl Graph {fn set_owner(&mut self,_:&Context)->Result{Ok(())}} struct ReaderHint; impl ReaderHint {fn has_readers(&self,count:&AtomicU32)->bool{count.load(Ordering::Acquire)!=0}} struct Queue { - owner:Option,installed:bool,growing:bool,growth_blocks:usize,free_list_generation:u64, + owner:Option,installed:bool,growing:bool,growth:Option,free_list_generation:u64,configuration:Option<()>,retained_owner:Option<()>, pool_phase:PoolPhase,release_sent_ns:Option,pool:Pool,binding:Option,previous:Option,graph:Graph, active:Fifo,replays:Fifo,quarantined:bool,released:bool,retirement_ready:bool, retired_by_teardown:bool,released_at:u64,exit:Option<()>,needs_fresh_backing:bool, source_hint:ReaderHint,source_readers:AtomicU32, } -fn idle(live:&Arc)->Queue{Queue{owner:Some(1),installed:true,growing:false,growth_blocks:0, - free_list_generation:0,pool_phase:PoolPhase::Populated,release_sent_ns:None,pool:Pool{pages:Some(Pages(10)),fail:false}, +fn idle(live:&Arc)->Queue{Queue{owner:Some(1),installed:true,growing:false,growth:None, + free_list_generation:0,configuration:None,retained_owner:None,pool_phase:PoolPhase::Populated,release_sent_ns:None,pool:Pool{pages:Some(Pages(10,21)),fail:false}, binding:Some(Binding{context:Context::new(live),pool_alias:Some(1),descriptors:Some(2)}),previous:None,graph:Graph, active:Fifo{len:0},replays:Fifo{len:0},quarantined:false,released:false,retirement_ready:true, retired_by_teardown:false,released_at:0,exit:None,needs_fresh_backing:false,source_hint:ReaderHint,source_readers:AtomicU32::new(0)}} @@ -53,8 +55,14 @@ def fixture(): const EAGAIN:i32=-11;const EFAULT:i32=-14; static CONSUMER:AtomicU32=AtomicU32::new(0); static PRODUCER:AtomicU32=AtomicU32::new(3); -struct Pages(u64); -impl Pages {fn page_list_va(&self)->u64{self.0}} +// A page list's address and its block count. +struct Pages(u64,usize); +struct PoolDescriptor; +impl Pages { + fn page_list_va(&self)->u64{self.0} + fn fits_capacity(&self,required:ComputeCapacity)->bool{let ComputeCapacity::Blocks(blocks)=required;self.1>=blocks} + fn descriptor(&self,_:u16)->Result{Ok(PoolDescriptor)} +} mod mmu {pub type KernelMapping=u64;} mod g17 { pub fn now_ns()->u64{crate::now_ns()} @@ -65,7 +73,7 @@ def fixture(): impl InitData { pub fn control_consumer(&self)->crate::Result<&crate::AtomicU32>{Ok(&crate::CONSUMER)} pub fn control_producer(&self)->crate::Result<&crate::AtomicU32>{Ok(&crate::PRODUCER)} - pub fn initialize_compute_pool_blocks(&self,_:u16,_:u64,_:usize)->crate::Result{if self.fail{Err(crate::EIO)}else{Ok(())}} + pub fn install_compute_pool(&self,_:crate::PoolDescriptor)->crate::Result{if self.fail{Err(crate::EIO)}else{Ok(())}} } } } @@ -103,19 +111,19 @@ def fixture(): let live=Arc::new(AtomicUsize::new(0));let init=initdata::InitData{fail:false}; for close_first in [false,true] {for consumed_first in [false,true] { let mut q=idle(&live);q.retirement_ready=true; - assert!(q.begin_growth(&init,40).unwrap());let token=q.growth_token().unwrap(); + assert!(q.begin_growth(&init,ComputeCapacity::Blocks(40)).unwrap());let token=q.growth_token().unwrap(); assert!(q.growing);assert!(q.pool_phase==PoolPhase::ReleaseSent(3)); assert_eq!(q.release_sent_ns,Some(now_ns()),"exact publication starts the release deadline"); assert_eq!(q.abort_growth(Growth{qid:8,phase:token.phase}).unwrap(),None,"wrong physical queue"); assert_eq!(q.abort_growth(Growth{qid:token.qid,phase:PoolPhase::ReleaseSent(4)}).unwrap(),None,"stale release transaction"); assert!(q.growing); - assert!(q.begin_growth(&init,40).is_err(),"another builder cannot steal growth"); + assert!(q.begin_growth(&init,ComputeCapacity::Blocks(40)).is_err(),"another builder cannot steal growth"); if close_first {q.release_owner().unwrap();} CONSUMER.store(if consumed_first{3}else{2},Ordering::SeqCst); assert!(q.observe_vacancy(CONSUMER.load(Ordering::SeqCst),3).unwrap().is_none(),"live builder owns growth"); // Timeout, epoch invalidation, notify/MMIO error all run this same abort. assert_eq!(q.abort_growth(token).unwrap(),Some(false)); - assert!(!q.growing);assert!(q.owner.is_none());assert_eq!(q.growth_blocks,0); + assert!(!q.growing);assert!(q.owner.is_none());assert_eq!(q.growth,None); assert_eq!(q.abort_growth(token).unwrap(),None); assert!(q.pool.pages.is_some(),"abort must retain backing"); assert_eq!(q.release_sent_ns,Some(now_ns()),"abort must not refresh or discard an owned deadline"); @@ -135,19 +143,23 @@ def fixture(): drop(registry);drop(deferred);assert_eq!(live.load(Ordering::SeqCst),0); }} for failure in 0..2 { - let mut q=idle(&live);q.retirement_ready=true;q.begin_growth(&init,40).unwrap(); + let mut q=idle(&live);q.retirement_ready=true;q.begin_growth(&init,ComputeCapacity::Blocks(40)).unwrap(); let token=q.growth_token().unwrap();CONSUMER.store(3,Ordering::SeqCst); - let mut pages=Some(Pages(20));let mut alias=Some(30); + let mut pages=Some(Pages(20,40));let mut alias=Some(30); q.pool.fail=failure==0; - assert!(q.finish_growth(&initdata::InitData{fail:failure==1},&mut pages,&mut alias,40).is_err()); + assert!(q.finish_growth(&initdata::InitData{fail:failure==1},&mut pages,&mut alias).is_err()); assert_eq!(pages.as_ref().unwrap().0,20,"failed rearm preserves tentative pages"); assert_eq!(q.pool.pages.as_ref().unwrap().0,10,"failed rearm preserves old pages"); assert_eq!(alias,Some(30));assert_eq!(q.binding.as_ref().unwrap().pool_alias,Some(1)); q.abort_growth(token).unwrap();assert!(q.observe_vacancy(3,3).unwrap().is_some()); } - let mut q=idle(&live);q.retirement_ready=true;q.begin_growth(&init,40).unwrap(); - let token=q.growth_token().unwrap();let mut pages=Some(Pages(20));let mut alias=Some(30); - let old=q.finish_growth(&init,&mut pages,&mut alias,40).unwrap(); + let mut q=idle(&live);q.retirement_ready=true;q.begin_growth(&init,ComputeCapacity::Blocks(40)).unwrap(); + let token=q.growth_token().unwrap();let mut pages=Some(Pages(20,39));let mut alias=Some(30); + // The fresh backing must hold what the growth began for. + assert_eq!(q.finish_growth(&init,&mut pages,&mut alias).err(),Some(EAGAIN),"undersized backing must not commit"); + assert!(q.growing);assert_eq!(q.pool.pages.as_ref().unwrap().0,10);assert_eq!(alias,Some(30)); + let mut pages=Some(Pages(20,40)); + let old=q.finish_growth(&init,&mut pages,&mut alias).unwrap(); assert_eq!(q.release_sent_ns,None,"committed replacement clears the witnessed release deadline"); assert_eq!(old._pages.unwrap().0,10);assert_eq!(old._alias,Some(1));assert!(pages.is_none());assert!(alias.is_none()); assert_eq!(q.pool.pages.as_ref().unwrap().0,20);assert_eq!(q.abort_growth(token).unwrap(),None,"committed growth cannot be aborted"); @@ -161,7 +173,8 @@ def main(): runtime = (ROOT / 'runtime.rs').read_text() assert 'growth = queue.growth_token();' in runtime and 'queue.abort_growth(token)' in runtime cases = [('fixed', source), - ('missing-abort-transfer', source.replace('self.growing = false;\n self.growth_blocks = 0;\n self.release_owner().map(Some)', 'self.release_owner().map(Some)')), + ('missing-abort-transfer', source.replace('self.growing = false;\n self.growth = None;\n self.release_owner().map(Some)', 'self.release_owner().map(Some)')), + ('ignore-growth-target', source.replace('if !pages.as_ref().ok_or(EINVAL)?.fits_capacity(target) {', 'if false {')), ('ignore-consumption', source.replace('if !crate::g17::freelist::control_consumed(consumer, producer, cursor)? {', 'if false {')), ('ignore-collector-pressure', source.replace('if !deferred.vacated_room() { break; }', ''))] with tempfile.TemporaryDirectory(prefix='g17-growth-') as tmp: @@ -169,6 +182,8 @@ def main(): path=Path(tmp)/(name+'.rs');path.write_text(code);binary=path.with_suffix('') subprocess.run([os.environ.get('RUSTC','rustc'),'--edition=2021','-Awarnings',str(path),'-o',str(binary)],check=True) result=subprocess.run([str(binary)],capture_output=True,text=True,timeout=15) + if name!='fixed': + assert code!=source,f'vacuous mutation: {name}' if name=='fixed': assert result.returncode==0,result.stderr print(result.stdout.strip()) diff --git a/tools/asahi/g17-owner-release-order-tests.py b/tools/asahi/g17-owner-release-order-tests.py index 55b95af1b279d..517eda49370d4 100755 --- a/tools/asahi/g17-owner-release-order-tests.py +++ b/tools/asahi/g17-owner-release-order-tests.py @@ -41,7 +41,7 @@ def check_guards(): prepare = (ROOT / 'runtime/render/prepare.rs').read_text() build = item(prepare, 'fn build_render_pair(') # The in-place rebind is refused while the pair's scheduler release is pending. - ordered(build, 'if !pair.reusable()?', 'firmware.queues.teardown.releasing(pair.context())', + ordered(build, 'if !pair.bindable(self.owner, &self.context)?', 'firmware.queues.teardown.releasing(pair.context())', 'return Err(EBUSY);', 'Build::Rebind {') rebind = item(prepare, 'fn rebind_pair(') # A live donor that closed while the binding was prepared is caught at commit. @@ -73,8 +73,8 @@ def check_guards(): 'return Err(EAGAIN);', 'queue.publish(') # A busy pair (for example one whose scheduler release is still being consumed) # kicks the event worker, which observes the release and parks the pair. - ordered(item(submit, 'fn wait_render_busy('), 'WORK_RING_WAIT_NS', - 'self.shared.queue_events();', 'self.wait_domain(') + ordered(item(submit, 'fn wait_render_busy('), 'self.wait_render_retirement(', 'WORK_RING_WAIT_NS') + ordered(item(submit, 'fn wait_render_retirement('), 'self.shared.queue_events();', 'self.wait_domain(') ordered(item(submit, 'fn wait_domain('), 'if remaining == 0', 'return Err(ETIMEDOUT);', 'domain.block(') borrowed = item((ROOT / 'runtime/render.rs').read_text(), 'fn take_pair(') ordered(borrowed, 'entry.borrowed_switch = pair.switch_in_flight();', @@ -102,8 +102,10 @@ def fixture(): 'pub(crate) fn owner_switch_from(', 'pub(crate) fn switch_in_flight(']).replace('pub(crate) ', '') record = render[render.index(' let scheduler = packet.context.scheduler_va();'): render.index(' self.executed = scheduler;') + len(' self.executed = scheduler;')] - idle = item((ROOT / 'runtime/render/teardown.rs').read_text(), - 'pub(in crate::g17::runtime) fn teardown_idle(').replace('pub(in crate::g17::runtime) ', '') + render_teardown = (ROOT / 'runtime/render/teardown.rs').read_text() + # teardown_idle() is idle_for() without the release-time kill exception. + idle = '\n'.join(item(render_teardown, name) for name in [ + 'pub(in crate::g17::runtime) fn teardown_idle(', 'fn idle_for(']).replace('pub(in crate::g17::runtime) ', '') compute_switch = item((ROOT / 'queue/compute.rs').read_text(), 'pub(crate) fn switching_from(').replace('pub(crate) ', '') compute_idle = item((ROOT / 'runtime/teardown.rs').read_text(), 'fn compute_idle(') @@ -116,6 +118,8 @@ def fixture(): impl Context { fn scheduler_va(&self) -> u64 { self.scheduler } fn is_published(&self) -> bool { self.published } + // Reached only for a release-time check, which this fixture does not make. + fn kill_published(&self) -> bool { unreachable!() } } #[derive(Clone, Copy, PartialEq, Eq)] enum Phase { Deferred, Draining, Released(u32) } diff --git a/tools/asahi/g17-ray-query-timeout-tests.sh b/tools/asahi/g17-ray-query-timeout-tests.sh index 9aaa3aa653c14..2ac2a7dc852a0 100755 --- a/tools/asahi/g17-ray-query-timeout-tests.sh +++ b/tools/asahi/g17-ray-query-timeout-tests.sh @@ -5,6 +5,18 @@ set -eu out=${1:?usage: g17-ray-query-timeout-tests.sh OUTPUT_DIRECTORY} root=$(CDPATH= cd -- "$(dirname -- "$0")/../.." && pwd) mkdir -p "$out" +# timeout.rs reads the driver's clock as crate::g17::now_ns(), so it builds +# as crate::g17::timeout beside a host clock, as it sits in the driver. +cat > "$out/g17-timeout-tests.rs" < u64 { + use std::time::{SystemTime, UNIX_EPOCH}; + SystemTime::now().duration_since(UNIX_EPOCH).map_or(1, |t| t.as_nanos() as u64) + } + #[path = "$root/drivers/gpu/drm/asahi/g17/timeout.rs"] + pub(crate) mod timeout; +} +RUST "${RUSTC:-rustc}" --edition=2021 --test \ - "$root/drivers/gpu/drm/asahi/g17/timeout.rs" -o "$out/g17-timeout-tests" + "$out/g17-timeout-tests.rs" -o "$out/g17-timeout-tests" "$out/g17-timeout-tests" diff --git a/tools/asahi/g17-resource-tests.py b/tools/asahi/g17-resource-tests.py index ae7e53ec6cf6f..f08622360d929 100644 --- a/tools/asahi/g17-resource-tests.py +++ b/tools/asahi/g17-resource-tests.py @@ -19,7 +19,11 @@ def item(start): while depth: depth+=(s[end]=='{')-(s[end]=='}');end+=1 return s[at:end] - constants='\n'.join(re.search(r'pub\(crate\) const '+name+r': [^;]+;',cfg).group(0) for name in ['RENDER_SLOTS','RENDER_PAIRS_PER_OWNER','RENDER_DEPTH']) + constants='\n'.join(re.search(r'pub\(crate\) const '+name+r': [^;]+;',cfg).group(0) for name in ['RENDER_SLOTS','RENDER_PAIRS_MAX','RENDER_DEPTH']) + pairs=(source/'g17/pairs.rs').read_text() + assert 'const G17D_PAIRS: u8 = RENDER_PAIRS_MAX as u8;' in pairs + assert 'PAIRS.store(if board.g17d_forms() { G17D_PAIRS } else { 1 }, Relaxed);' in pairs + at=pairs.index('const SIBLING_BACKLOG');backlog=pairs[at:pairs.index('\n}\n',pairs.index('fn sibling_backlog',at))+3] mmu=(source/'mmu.rs').read_text() limits='\n'.join(re.search(r'const '+name+r': [^;]+;',mmu).group(0) for name in ['EXECUTION_CONTEXT_MASK','MAX_EXECUTION_CONTEXTS']) header='''#![allow(dead_code, unused_macros)] @@ -27,51 +31,67 @@ def item(start): mod g17 { pub mod trace { pub fn teardown(_:u8,_:u64,_:u64,_:u64) {} } pub mod capacity { pub enum Site { RenderSlots, LogicalQueues, ComputeReserve, PhysicalQids } pub fn refused(_: Site) -> i32 { -28 } + } pub mod pairs { use std::sync::atomic::{AtomicU8, Ordering::Relaxed}; + '''+backlog+''' }} const ENOSPC:i32=-28; const EIO:i32=-5; macro_rules! resource_exhausted { ($s:literal)=>{ENOSPC}; } type KVec = Vec; struct InFlight; mod cfg {\n'''+constants+'}\n'+limits+'\n' - methods=['const fn new','fn exhausted','fn refusal','fn advance','fn take_idle','fn acquire','fn claim_sibling','fn reassign_unused_spare','fn warmable_sibling','fn borrow_warm_sibling','fn cancel_empty_slot','fn release'] + methods=['const fn new','fn exhausted','fn owner_quarantine','fn refusal','fn advance','fn take_idle','fn acquire','fn usable','fn claim_sibling','fn reassign_unused_spare','fn warmable_sibling','fn borrow_warm_sibling','fn cancel_empty_slot','fn release'] code=header+item('struct State')+'\nimpl State {\n'+re.search(r' const MASK: u64 = .*;',s).group(0)+'\n'+'\n'.join(item(n) for n in methods)+'\n}\n'+''' + // A queue holds g17::pairs::per_owner() pairs (Backend::pairs): one, or + // RENDER_PAIRS_MAX on G17D. Both admissions run. fn main(){ - if cfg::RENDER_PAIRS_PER_OWNER == 1 { + one_pair(1); + two_pairs(cfg::RENDER_PAIRS_MAX); + } + fn one_pair(pairs:usize){ let mut state=State::new(); let mut slots=Vec::new(); for owner in 1..=u64::from(MAX_EXECUTION_CONTEXTS) { - let slot=state.acquire(owner).expect("every logical owner must fit");slots.push(slot); + let slot=state.acquire(owner,pairs).expect("every logical owner must fit");slots.push(slot); assert_eq!(state.owners.iter().filter(|id|**id==owner).count(),1); } for slot in slots {state.release(slot);} - let mut state=State::new();let first=state.acquire(1).unwrap(); - for _ in 1..cfg::RENDER_DEPTH {assert_eq!(state.acquire(1),Some(first));} - assert_eq!(state.acquire(1),None);state.release(first);assert_eq!(state.acquire(1),Some(first)); + let mut state=State::new();let first=state.acquire(1,pairs).unwrap(); + for _ in 1..cfg::RENDER_DEPTH {assert_eq!(state.acquire(1,pairs),Some(first));} + assert_eq!(state.acquire(1,pairs),None);state.release(first);assert_eq!(state.acquire(1,pairs),Some(first)); println!("shared render admission: all58 owners fit; depth16 waits for exact retirement without consuming a spare"); - return; - } + } + fn two_pairs(pairs:usize){ + // 32 owners fill the 63 usable slots: 31 with a sibling, the last without. + assert_eq!(pairs,2); let mut state=State::new(); for owner in 1..=32 { assert!(state.refusal(owner).is_ok()); - let slot=state.acquire(owner).expect("existing owner fits"); + let slot=state.acquire(owner,pairs).expect("existing owner fits"); state.release(slot); } assert_eq!(state.active,0,"all submitted work retired"); let retained=state.owners.iter().filter(|o|**o!=0).count(); assert_eq!(retained,63); assert_eq!(state.refusal(33),Err(ENOSPC)); - assert!(state.acquire(33).is_none()); + assert!(state.acquire(33,pairs).is_none()); println!("confirmed: 32 open idle owners retain all 63 usable pairs; owner33 receives ENOSPC"); // Yield one unused spare for each additional logical owner. The runtime // separately proves that these physical graphs have never been published. for owner in 33..=u64::from(MAX_EXECUTION_CONTEXTS) { let slot=(1..cfg::RENDER_SLOTS).find(|slot|state.reassign_unused_spare(*slot as u8,None,Some(owner))).unwrap(); - assert_eq!(state.acquire(owner),Some(slot as u8));state.release(slot as u8); + assert_eq!(state.acquire(owner,pairs),Some(slot as u8));state.release(slot as u8); } - for owner in 1..=u64::from(MAX_EXECUTION_CONTEXTS) {let slot=state.acquire(owner).unwrap();state.release(slot);} + for owner in 1..=u64::from(MAX_EXECUTION_CONTEXTS) {let slot=state.acquire(owner,pairs).unwrap();state.release(slot);} assert!(state.owners.iter().all(|owner|*owner<=u64::from(MAX_EXECUTION_CONTEXTS))); println!("fixed: all{} logical owners render and resume without reducing slot count or depth",MAX_EXECUTION_CONTEXTS); - let mut guarded=State::new();let first=guarded.acquire(1).unwrap();guarded.release(first); + let mut depth=State::new();let first=depth.acquire(1,pairs).unwrap(); + for _ in 1..pairs*usize::from(cfg::RENDER_DEPTH) {assert!(depth.acquire(1,pairs).is_some());} + assert_eq!(depth.acquire(1,pairs),None); + let held:Vec<_>=(0..cfg::RENDER_SLOTS).filter(|slot|depth.owners[*slot]==1).collect(); + assert_eq!(held.len(),pairs);assert!(held.iter().all(|slot|depth.refs[*slot]==cfg::RENDER_DEPTH)); + depth.release(first);assert_eq!(depth.acquire(1,pairs),Some(first)); + println!("two pairs: depth {} over both pairs, then waits for exact retirement",pairs*usize::from(cfg::RENDER_DEPTH)); + let mut guarded=State::new();let first=guarded.acquire(1,pairs).unwrap();guarded.release(first); let spare=guarded.owners.iter().enumerate().find(|(s,o)|**o==1&&*s!=first as usize).unwrap().0; guarded.take_idle(spare,1); assert!(!guarded.reassign_unused_spare(spare as u8,Some(1),Some(2)),"warm borrower excludes reclamation"); @@ -86,23 +106,32 @@ def item(start): assert!(!guarded.reassign_unused_spare(64,None,Some(3))); guarded.released |= 1u64<bool { false } } impl ControlRecord { pub fn bytes(&self)->&[u8] { &[] } @@ -80,6 +81,11 @@ def item(source, signature): pub fn asserts_gpu_power(&self)->bool { false } } } + // M5's statistics tags and power rule, as the family methods below read them. + pub mod m5 { + M5_STATISTICS_TYPES + pub mod power { M5_ASSERTS_GPU_POWER } + } pub mod initdata { pub mod status { pub const TRACE_STATE:u32=0; pub const TRACE_RING:u32=0x100; @@ -94,9 +100,11 @@ def item(source, signature): pub struct InitData { pub cursors:[[AtomicU32;2];2], pub slots:RefCell<[[[u32;18];8];2]>, + pub m5:bool, } impl InitData { - pub fn new()->Self { Self { cursors:std::array::from_fn(|_|std::array::from_fn(|_|AtomicU32::new(0))), slots:RefCell::new([[[0;18];8];2]) } } + pub fn new()->Self { Self { cursors:std::array::from_fn(|_|std::array::from_fn(|_|AtomicU32::new(0))), slots:RefCell::new([[[0;18];8];2]), m5:false } } + INITDATA_FAMILY pub fn push(&self,role:Role,words:[u32;18]) { use std::sync::atomic::Ordering::Relaxed; let producer=self.cursors[role as usize][1].load(Relaxed); @@ -667,8 +675,20 @@ def main(): policy = '\n'.join(item(stats_rs, signature) for signature in ['pub(super) const fn max_age_ns', 'pub(super) const fn reports_pstate', 'pub(super) fn qualify(']) + # The rings take the family's statistics tags and control power rule from + # InitData; build those production methods into the fixture's InitData. + initdata = (DRIVER / 'g17/initdata.rs').read_text() + family = '\n'.join(item(initdata, signature).replace('pub(super) ', 'pub ', 1) for signature in + ['pub(super) fn statistics_types(', 'pub(super) fn control_asserts_gpu_power(']) + m5 = (DRIVER / 'g17/fw/m5/mod.rs').read_text() + m5_types = next(x for x in m5.splitlines() if 'const STATISTICS_TYPES:' in x).replace('pub(crate)', 'pub') + m5_power = item((DRIVER / 'g17/fw/m5/power.rs').read_text(), + 'pub(crate) const fn asserts_gpu_power(').replace('pub(crate)', 'pub', 1) + header = (RUST_HEADER.replace('M5_STATISTICS_TYPES', m5_types).replace('M5_ASSERTS_GPU_POWER', m5_power) + .replace('INITDATA_FAMILY', family)) + reader_rs = 'pub(crate) struct Reader(pub(super) Arc);\n' + item(stats_rs, 'impl Reader') - (work / 'rings.rs').write_text(RUST_HEADER + 'mod stats { use super::{fw::variant::InitGraph, telemetry::Readout, Shared};\n' + (work / 'rings.rs').write_text(header + 'mod stats { use super::{fw::variant::InitGraph, telemetry::Readout, Shared};\n' + 'use std::sync::{Arc,atomic::Ordering};\n' + reader_rs + '\n' + policy + '\n}\nmod telemetry;\nmod channel;\n' + RUST_TESTS) # Rust resolves child modules relative to the inline parent's directory. diff --git a/tools/asahi/g17-unpublished-tests.py b/tools/asahi/g17-unpublished-tests.py index f06eedf7f64f8..594621bebe661 100644 --- a/tools/asahi/g17-unpublished-tests.py +++ b/tools/asahi/g17-unpublished-tests.py @@ -46,7 +46,7 @@ def fixture(): let context=q.binding.as_ref().unwrap().context.clone(); if predecessor!=0 { let old=Context::new(100+owner,&live); - q.previous=Some(Previous{binding:if predecessor==1{Some(Binding{context:old.clone()})}else{None},context:old,publication:None}); + q.previous=Some(Previous{binding:if predecessor==1{Some(Binding{context:old.clone()})}else{None},context:old,publication:None,moved:false}); } channels.push(Backend{q:std::cell::RefCell::new(q),owner,context}); } @@ -64,12 +64,12 @@ def fixture(): match guard { 0=>q.unpublished_binding=false, 1=>q.exit=Some(Exit{phase:Phase::Acknowledged}), - 2=>q.active.push(Active{packet:Arc::new(Packet{completion:Arc::new(Completion)}),ticket:None,qos_pending:false}).unwrap(), + 2=>q.active.push(Active{packet:Arc::new(Packet{completion:Arc::new(Completion)}),ticket:None,observed:None,qos_pending:false}).unwrap(), 3=>q.replays.push(Arc::new(Packet{completion:Arc::new(Completion)})).unwrap(), 4=>q.growing=true, 5=>q.quarantined=true, 6=>q.awaiting_witness=true, - 7=>q.previous=Some(Previous{context:context.clone(),binding:None,publication:Some((1,1))}), + 7=>q.previous=Some(Previous{context:context.clone(),binding:None,publication:Some((1,1)),moved:false}), 8=>q.pool_phase=PoolPhase::ReleaseSent(3), 9=>owner=3, 10=>ctx=Context::new(4,&live), @@ -91,7 +91,9 @@ def fixture(): def main(): source=fixture() compute=(ROOT/'queue/compute.rs').read_text() - assert compute.count('visible = true;\n self.unpublished_binding = false;')==2 + # Every publication path (G17P shared, G17P, M5) clears the unpublished + # binding exactly when its record becomes firmware-visible. + assert compute.count('visible = true;\n self.unpublished_binding = false;')==3 backend=(ROOT/'runtime/backend.rs').read_text() assert 'if !lease.is_current() { drop(state); drop(lease); continue; }' in backend no_kill, count = re.subn(r'\|\|\s*self\.exit\.is_some\(\)\s*(?=\|\|\s*self\.active\.len\s*!=\s*0)', '', source) diff --git a/tools/asahi/neo-auxiliary-tests.py b/tools/asahi/neo-auxiliary-tests.py index 68236fe19d192..2fc8ada61d93a 100644 --- a/tools/asahi/neo-auxiliary-tests.py +++ b/tools/asahi/neo-auxiliary-tests.py @@ -79,15 +79,20 @@ def main(): args.output.mkdir(parents=True, exist_ok=True) source = (subprocess.check_output(['git','show','990a45b609dd:'+PATH.replace('/asahi/', '/asahi-neo/')], cwd=ROOT,text=True) if args.old else (ROOT/PATH).read_text()) - structure = item(source, 'pub(super) struct Generation').replace('pub(super)', '') - constructor = item(source[source.index('impl Generation'):], ' fn new(') - addresses = item(source, ' pub(super) fn addresses').replace('pub(super)', '') + # The G17P generation is NeoGeneration since the M5 one joined it behind + # the Generation enum; the old revision still calls it Generation. + generation = 'Generation' if args.old else 'NeoGeneration' + structure = item(source, 'pub(super) struct ' + generation + ' ').replace('pub(super)', '') + implementation = source[source.index('impl ' + generation + ' {'):] + constructor = item(implementation, ' fn new(') + addresses = item(implementation, ' pub(super) fn addresses').replace('pub(super)', '') storage=(ROOT/'drivers/gpu/drm/asahi/g17/fw/render/storage.rs').read_text() constants=[] for name in ['STATE_SIZE','DEFLAKE','TA_STATUS','AUXILIARY','AUXILIARY_SIZE']: line=next(x for x in storage.splitlines() if 'const '+name+':' in x) constants.append(line.replace('pub(crate)','pub')) - code=PREFIX.replace('CONSTANTS','\n'.join(constants))+structure+'\nimpl Generation{'+constructor+addresses+'}\n'+MAIN + code=(PREFIX.replace('CONSTANTS','\n'.join(constants))+structure+'\nimpl '+generation+'{'+constructor+addresses+'}\n' + +('' if generation=='Generation' else 'type Generation='+generation+';\n')+MAIN) if args.mutation: code=code.replace('self.state.gpu_va() + storage::AUXILIARY as u64','self.state.gpu_va()') path=args.output/'controls.rs';path.write_text(code) diff --git a/tools/asahi/queue-priority-tests.py b/tools/asahi/queue-priority-tests.py index 1c5b4695c85f7..7fd7647234de4 100644 --- a/tools/asahi/queue-priority-tests.py +++ b/tools/asahi/queue-priority-tests.py @@ -7,7 +7,8 @@ root = Path(__file__).resolve().parents[2] s = (root / 'drivers/gpu/drm/asahi/file.rs').read_text() a = s.index(' if data.flags != 0', s.index('pub(crate) fn queue_create(')) -b = s.index(' let queues_xa', a) +# Keep this credential/priority fixture scoped before queue-quota accounting. +b = s.index(' let share =', a) gate = s[a:b] predicate = (root / 'drivers/gpu/drm/asahi/m3/m3_drm.rs').read_text().split('pub(crate) fn supports_high_priority(&self) -> bool {', 1)[1].split('\n }', 1)[0] submit = (root / 'drivers/gpu/drm/asahi/m3/m3_submit.rs').read_text() diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/README b/tools/testing/selftests/drivers/gpu/apple-afkv2/README index 0b0423fbb8e4e..f30cbb2ee12f2 100644 --- a/tools/testing/selftests/drivers/gpu/apple-afkv2/README +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/README @@ -15,9 +15,11 @@ correlation, token preservation and command ID packing. The owned-copy test extracts the actual production function from afk.c, prints its SHA256, and compiles it with synthetic command slots and CPU buffers. It -checks active/inactive/done slots, high device addresses, boundaries and copies. -Locks and DMA barriers are mocked: this test does not qualify concurrency, -DMA coherency, firmware, a graphics device or any hardware display behavior. +checks active/inactive/done slots, high device addresses, boundaries and copies, +and that only firmware 27 hands an unowned buffer to its retained-DART path. +Locks, DMA barriers and that path (afk_v27_access_remote()) are mocked: this +test does not qualify concurrency, DMA coherency, firmware, a graphics device +or any hardware display behavior. All build products are temporary. No device node is opened and no GPU, firmware or target machine is contacted. The extractor fails if production function diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/owned-prefix.c b/tools/testing/selftests/drivers/gpu/apple-afkv2/owned-prefix.c index 8ff88ec1ec5b0..49d9340e1fd43 100644 --- a/tools/testing/selftests/drivers/gpu/apple-afkv2/owned-prefix.c +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/owned-prefix.c @@ -38,10 +38,40 @@ struct epic_cmd_info { size_t rxlen, txlen; }; +enum dcp_firmware_version { + DCP_FIRMWARE_UNKNOWN, + DCP_FIRMWARE_V_27_0, +}; + +struct apple_dcp { + enum dcp_firmware_version fw_compat; +}; + struct apple_dcp_afkep { bool wire_v2; + struct apple_dcp *dcp; }; +/* Firmware 27's retained-DART fallback: record the call, return the verdict. */ +static int remote_calls, remote_ret = -EACCES; +static u64 remote_dva; +static size_t remote_size; +static bool remote_to_remote, remote_copy; + +static int afk_v27_access_remote(struct apple_dcp_afkep *ep, u64 dva, + void *buffer, size_t size, bool to_remote, + bool copy) +{ + (void)ep; + (void)buffer; + remote_calls++; + remote_dva = dva; + remote_size = size; + remote_to_remote = to_remote; + remote_copy = copy; + return remote_ret; +} + struct apple_epic_service { int lock; struct apple_dcp_afkep *ep; diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/owned-suffix.c b/tools/testing/selftests/drivers/gpu/apple-afkv2/owned-suffix.c index 770b16bc2fa94..f5cbfc7396088 100644 --- a/tools/testing/selftests/drivers/gpu/apple-afkv2/owned-suffix.c +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/owned-suffix.c @@ -3,7 +3,8 @@ int main(void) { u8 rx[16], tx[16], output[16]; - struct apple_dcp_afkep ep = { .wire_v2 = true }; + struct apple_dcp dcp = { .fw_compat = DCP_FIRMWARE_UNKNOWN }; + struct apple_dcp_afkep ep = { .wire_v2 = true, .dcp = &dcp }; struct apple_epic_service service = { .ep = &ep }; struct epic_cmd_info *cmd = &service.cmds[0]; @@ -51,6 +52,24 @@ int main(void) false) == 0); assert(afk_v2_access_owned(&service, cmd->rxbuf_dma + 1, NULL, 16, true, false) == -EACCES); - puts("PASS owned-copy boundaries (actual extracted function; lock/barrier mocks)"); + /* Only firmware 27 falls back to its retained mappings, and only when unowned. */ + assert(remote_calls == 0); + dcp.fw_compat = DCP_FIRMWARE_V_27_0; + assert(afk_v2_copy_owned(&service, cmd->rxbuf_dma + 12, output, 4, + false) == 0); + assert(remote_calls == 0); + assert(afk_v2_copy_owned(&service, 0x20000003000ULL, output, 4, true) == + -EACCES); + assert(remote_calls == 1 && remote_dva == 0x20000003000ULL && + remote_size == 4 && remote_to_remote && remote_copy); + remote_ret = 0; + assert(afk_v2_access_owned(&service, 0x20000003000ULL, NULL, 8, false, + false) == 0); + assert(remote_calls == 2 && remote_size == 8 && !remote_to_remote && + !remote_copy); + assert(afk_v2_copy_owned(&service, 0x20000003000ULL, NULL, 1, false) == + -EINVAL); + assert(remote_calls == 2); + puts("PASS owned-copy boundaries and firmware-27 fallback (actual extracted function; lock/barrier and fallback mocks)"); return 0; } diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/desktop.py b/tools/testing/selftests/drivers/gpu/apple-dcp/desktop.py index a7f88da81652e..d42cc06149b66 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/desktop.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/desktop.py @@ -221,6 +221,7 @@ def brightness_takeover(root): #define WRITE_ONCE(x,v) ((x)=(v)) #define DCP_FIRMWARE_V_26_6_DESKTOP 5 #define DCP_FIRMWARE_H17P 1 +#define DCP_FIRMWARE_V_27_0 6 #define DCP_IOMFB_METHODS_H17G 7 #define MAX_BRIGHTNESS_PART2 510U #define BACKLIGHT_PLATFORM 1 @@ -397,7 +398,8 @@ def panel_identity(root, output=None): backend = (root / 'drivers/gpu/drm/apple/iomfb_v26_6_desktop.c').read_text() dictionary = block((root/'drivers/gpu/drm/apple/parser.c').read_text(), 'static int parse_panel_edid_value(') dictionary += block((root/'drivers/gpu/drm/apple/parser.c').read_text(), 'int parse_panel_edid_data(') - dictionary += block(backend, 'struct dcp_dict_passthrough_req_v26_6 {')+';\n' + # The dictionary record is shared with firmware 27's analytics event. + dictionary += block((root/'drivers/gpu/drm/apple/iomfb.h').read_text(), 'struct dcp_dict_passthrough_req {')+';\n' dictionary += block(backend, 'static bool trampoline_panel_dictionary_v26_6(') dictionary += block(backend, 'static bool iomfb_check_callback_exact_v26_6_desktop(') dictionary += '\n#define iomfb_check_callback_v26_6_desktop iomfb_check_callback_exact_v26_6_desktop\n' @@ -462,7 +464,7 @@ def panel_identity(root, output=None): }; for(unsigned form=0;form<3;form++) { desktop=(__typeof__(desktop)){0};notifications=0; - struct dcp_dict_passthrough_req_v26_6 wire={0};u32 result=0; + struct dcp_dict_passthrough_req wire={0};u32 result=0; strcpy((char *)wire.key,form?"ProductAttributes":"DisplayAttributes"); const u8 *data=form==2?minimal:form?product:attributes; size_t size=form==2?sizeof(minimal):form?sizeof(product):sizeof(attributes); @@ -595,6 +597,7 @@ def powerlog_framing(root): prefix += r''' #define DCP_FIRMWARE_H17P 1 #define DCP_FIRMWARE_V_26_6_DESKTOP 5 +#define DCP_FIRMWARE_V_27_0 6 #define DCP_IOMFB_METHODS_H17G 7 #define READ_ONCE(x) (x) #define WRITE_ONCE(x,v) ((x)=(v)) @@ -663,10 +666,18 @@ def powerlog_framing(root): d.fw_compat=DCP_FIRMWARE_H17P; afk_recv_compact_message(&ep,3,wire,sizeof(wire)-8); assert(hints==3 && !service.users); + d.fw_compat=DCP_FIRMWARE_V_27_0; + afk_recv_compact_message(&ep,3,wire,sizeof(wire)-8); + assert(hints==3 && !service.users); d.fw_compat=DCP_FIRMWARE_V_26_6_DESKTOP; /* The first key without its value gives no reading. */ afk_recv_compact_message(&ep,3,wire,24+12); assert(hints==3 && !service.users); + /* Firmware 27 takes a framed report's hint as H17P does. */ + d.fw_compat=DCP_FIRMWARE_V_27_0;hint=0; + memcpy(wire+16,report,sizeof(report)); + afk_recv_compact_message(&ep,3,wire,16+sizeof(report)); + assert(hints==4 && hint==123000 && !service.users); return 0; } ''' diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/edid.py b/tools/testing/selftests/drivers/gpu/apple-dcp/edid.py index 5caff361a38ca..50f6bd6f89166 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/edid.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/edid.py @@ -53,6 +53,7 @@ def function(file,signature): /* The T6030 14.7 backend never matches these H17P scenarios. */ #define DCP_FIRMWARE_V_14_7 (DCP_FIRMWARE_H17P + 1) #define DCP_FIRMWARE_V_14_7_TEMPLATE (DCP_FIRMWARE_H17P + 2) +#define DCP_FIRMWARE_V_27_0 (DCP_FIRMWARE_H17P + 3) #define EPIC_TYPE_NOTIFY 0 #define EPIC_CAT_REPORT 0 #define max(a,b) ((a)>(b)?(a):(b)) @@ -81,7 +82,7 @@ def function(file,signature): static bool __attribute__((unused)) cancel_work_sync(struct work_struct *w) { bool queued=w->pending;assert(!queued);return queued; } struct apple_dcp;struct apple_dcp_afkep;struct apple_epic_service; void afk_service_retry(struct apple_dcp_afkep *ep); -struct apple_epic_service_ops { char name[32];bool reusable,optional;void (*init)(struct apple_epic_service *,const char *,const char *,s64);int (*prepare)(struct apple_epic_service *,const char *,const char *,s64);int (*revalidate)(struct apple_epic_service *,const char *,const char *,s64);void (*teardown)(struct apple_epic_service *); }; +struct apple_epic_service_ops { char name[32];bool reusable,optional;void (*init)(struct apple_epic_service *,const char *,const char *,s64);int (*prepare)(struct apple_epic_service *,const char *,const char *,s64);int (*revalidate)(struct apple_epic_service *,const char *,const char *,s64);void (*init_props)(struct apple_epic_service *,s64,const void *,size_t);void (*teardown)(struct apple_epic_service *); }; struct epic_cmd_info { u16 tag;u8 type;size_t reply_length,reply_len;bool reply_len_valid;void *rxbuf,*txbuf;dma_addr_t rxbuf_dma,txbuf_dma;size_t rxlen,txlen;u32 retcode;bool done,free_on_ack;int *completion; }; struct apple_epic_service { struct apple_dcp_afkep *ep;const struct apple_epic_service_ops *ops; @@ -467,13 +468,21 @@ def function(file,signature): optional_admission_controls(); struct item h17p[]={{"compartment","dcpav-service-epic",0},{"name","AppleDCPDP2HDMI",0},{"interface-id",NULL,13}}; struct dcp_parse_ctx parse_ctx={.items=h17p,.count=3};const char *parsed_name,*parsed_class;s64 parsed_unit=-1; - assert(!parse_epic_service_init(&parse_ctx,&parsed_name,&parsed_class,&parsed_unit,true)); + assert(!parse_epic_service_init(&parse_ctx,&parsed_name,&parsed_class,&parsed_unit,true,false)); assert(parsed_unit==13 && !strcmp(parsed_name,name) && !strcmp(parsed_class,class)); struct apple_epic_service *proof=prepare(13,parsed_unit,parsed_name,parsed_class);assert(proof);afk_service_discard(proof); free((void *)parsed_name);free((void *)parsed_class); struct item legacy[]={{"EPICName","dcpav-service-epic",0},{"EPICProviderClass","legacy-class",0},{"EPICUnit",NULL,0}}; parse_ctx=(struct dcp_parse_ctx){.items=legacy,.count=3}; - assert(!parse_epic_service_init(&parse_ctx,&parsed_name,&parsed_class,&parsed_unit,false));assert(parsed_unit==0); + assert(!parse_epic_service_init(&parse_ctx,&parsed_name,&parsed_class,&parsed_unit,false,false));assert(parsed_unit==0); + free((void *)parsed_name);free((void *)parsed_class); + /* Firmware 27 carries both spellings; its EPICUnit is the unit in either order. */ + struct item both[]={{"compartment","dcpav-service-epic",0},{"name","AppleDCPDP2HDMI",0},{"EPICUnit",NULL,5},{"interface-id",NULL,13}}; + parse_ctx=(struct dcp_parse_ctx){.items=both,.count=4}; + assert(!parse_epic_service_init(&parse_ctx,&parsed_name,&parsed_class,&parsed_unit,true,true));assert(parsed_unit==5); + free((void *)parsed_name);free((void *)parsed_class); + parse_ctx=(struct dcp_parse_ctx){.items=both,.count=4}; + assert(!parse_epic_service_init(&parse_ctx,&parsed_name,&parsed_class,&parsed_unit,true,false));assert(parsed_unit==13); free((void *)parsed_name);free((void *)parsed_class); ep.wq=&ep;ep.command_tags=calloc(AFKV2_INTERFACES,1);assert(ep.command_tags); struct apple_epic_service *old=prepare(1,1,name,class);assert(old);publish(old,1); @@ -1025,6 +1034,7 @@ def startup(root): #define DCP_IOMFB_METHODS_H17P_EXTERNAL 2 #define DPAVSERV_ENDPOINT 0x28 #define DPAVSERV_ENDPOINT_V2 0x2a +#define DCP_FIRMWARE_V_27_0 7 #define IS_ERR(p) ((uintptr_t)(p) > (uintptr_t)-4096) #define PTR_ERR(p) ((int)(intptr_t)(p)) #define ERR_PTR(e) ((void *)(intptr_t)(e)) @@ -1041,6 +1051,7 @@ def startup(root): struct { int iomfb_method_profile; bool fw_14_7_endpoints; } hw; bool ext_dead,external_native; struct { unsigned enable_completion; bool enabled; } dcpavserv; struct apple_dcp_afkep *dcpavservep; + int fw_compat; }; static const unsigned dpavservep_ops[] = {0}; static struct apple_dcp_afkep endpoint; @@ -1100,7 +1111,14 @@ def startup(root): init_error=0; } } - puts("PASS optional legacy discovery, AFKV2 early/absent service, endpoint failures"); + /* Firmware 27 speaks AFKV2 on the legacy endpoint and does not wait. */ + struct apple_dcp v27={.fw_compat=DCP_FIRMWARE_V_27_0}; + announce=false;waits=warnings=starts=shutdowns=detaches=debug=0; + assert(dpavservep_init(&v27)==0); + assert(starts==1&&!waits&&!warnings&&!shutdowns&&!detaches&&!debug); + assert(v27.dcpavservep==&endpoint&&endpoint.wire_v2&&!endpoint.inline_v2); + assert(endpoint.endpoint==DPAVSERV_ENDPOINT&&!v27.dcpavserv.enabled); + puts("PASS optional legacy discovery, AFKV2 early/absent service, firmware 27, endpoint failures"); return 0; } ''' diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py b/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py index 0bf6edeac8c19..f459d57f5e475 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py @@ -1200,9 +1200,17 @@ def block(marker): struct device *dev; struct apple_dcp_hw_data hw; struct resource *disp_registers[10],disp_bw_scratch_res,disp_bw_doorbell_res; u32 disp_bw_scratch_index,disp_bw_scratch_offset,disp_bw_doorbell_index,nr_disp_registers; + u64 disp_firmware_base[MAX_DISP_REGISTERS]; }; +#define IS_ALIGNED(x, a) (((x) & ((a) - 1)) == 0) +#define U64_MAX UINT64_MAX static bool of_property_present(struct device_node *n,const char *name) { return !strcmp(name,"reg-names") && n->named; } +/* Firmware-physical windows follow the pmgr scratch lookup, which fails here. */ +static int of_property_count_u64_elems(struct device_node *n,const char *name) +{ (void)n;(void)name;assert(!"firmware-physical-regs outside the T8140 profiles");return -EINVAL; } +static int of_property_read_u64_array(struct device_node *n,const char *name,u64 *v,unsigned count) +{ (void)n;(void)name;(void)v;(void)count;assert(!"firmware-physical-regs outside the T8140 profiles");return -EINVAL; } static int of_property_count_strings(struct device_node *n,const char *name) { assert(!strcmp(name,"reg-names"));return n->count+1; } static int of_property_read_string_index(struct device_node *n,const char *name,int i,const char **v) @@ -1978,6 +1986,7 @@ def dispatcher(root): #define WRITE_ONCE(x,v) ((x)=(v)) #define DCP_FIRMWARE_H17P 1 #define DCP_FIRMWARE_V_26_6_DESKTOP 5 +#define DCP_FIRMWARE_V_27_0 6 #define DCP_IOMFB_METHODS_H17P_EXTERNAL 2 #define DCP_IOMFB_METHODS_H17G 3 #define DCP_MAX_CALL_DEPTH 8 @@ -2014,6 +2023,9 @@ def dispatcher(root): { (void)s; return n->external; } static bool iomfb_check_callback_h17p(struct apple_dcp *d, int t, u32 i, u32 o) { (void)d; (void)t; (void)i; (void)o; return true; } +/* Only handled callbacks are size-checked; this fixture registers none. */ +static bool iomfb_v27_callback_size_valid(unsigned t, u32 i, u32 o) +{ (void)t; (void)i; (void)o; assert(false); return false; } static bool iomfb_uses_queue(struct apple_dcp *d) { (void)d; return false; } static bool test_and_set_bit(int t, unsigned long *b) { (void)t; (void)b; return false; } static void complete(int *p) { *p = 1; } diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/m4.py b/tools/testing/selftests/drivers/gpu/apple-dcp/m4.py index 622957c3b6837..cd855c0f25dea 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/m4.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/m4.py @@ -656,6 +656,7 @@ def blank(root): #define DRM_FORMAT_XRGB8888 0x34325258U #define DRM_FORMAT_ARGB8888 0x34325241U #define DCP_FIRMWARE_H17P 1 +#define DCP_FIRMWARE_V_27_0 3 #define DCP_IOMFB_METHODS_H17G 2 #define READ_ONCE(x) (x) #define spin_lock_irqsave(p,f) do { (void)(p); (f)=0; } while(0) diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/rate.py b/tools/testing/selftests/drivers/gpu/apple-dcp/rate.py index c4c92d244f377..f3bfbc3954b15 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/rate.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/rate.py @@ -52,6 +52,11 @@ def wire(value, out): header = header[header.index('struct apple_dcp;'):header.index('int parse(')] refresh = (base / 'dcp-refresh.h').read_text().replace('#include ', '') header = refresh + '\n' + header + # parse_dptx_caps() fills the firmware-27 DPTX capabilities parser.c + # takes from dcp-peer-v27.h. + peer_v27 = base / 'dcp-peer-v27.h' + if peer_v27.exists(): + header += block(peer_v27.read_text(), 'struct dcp_dptx_caps {') + ';\n' # Header helpers stay header-like: unused here, as in any includer. header = header.replace('static inline ', 'static inline __attribute__((unused)) ') edid_header = (root / 'include/drm/drm_edid.h').read_text() @@ -1190,6 +1195,8 @@ def hotplug(root): #define trace_dptxport_call_set_link_rate(...) ((void)0) /* Not a T8132 dcpext: its DRM outputs never follow this link. */ static bool dcp_is_t8132_ext(const struct apple_dcp *d) { (void)d; return false; } +/* The J700 lane policy keys on the external H17P profile, not the AFK v2 codec. */ +static bool dcp_is_external_h17p(struct apple_dcp *d) { return d->hw.iomfb_method_profile==2; } struct device; static void t8132_ext_drm_hotplug(struct device *dcpext) { (void)dcpext; assert(!"T8132 dcpext hotplug in an H17P scenario"); } static unsigned phy_capacity=2, phy_calls, hardware_rate, hardware_lanes; @@ -1229,10 +1236,11 @@ def hotplug(root): bool external_physically_absent,main_display,typec_crtc_off,external_cable,external_error,external_power_attempted,external_stopped; bool external_hotplug_pending,external_hotplug_connected,typec_poweron_hpd; u64 external_stop_generation,external_rx_generation,external_crtc_generation; - unsigned swap_watchdog_retrains;int typec_iomfb_hpd_ready;''') + unsigned swap_watchdog_retrains;int typec_iomfb_hpd_ready,typec_display_ready;''') stubs = r''' #define DCP_HOTPLUG_NOTIFY 1 -static bool dcp_is_external_h17p(struct apple_dcp *d) { return d->hw.iomfb_method_profile==2; } +/* Firmware 27's resume readiness; this H17P display never runs it. */ +#define DCP_FIRMWARE_V_27_0 (DCP_FIRMWARE_H17P + 5) static bool dcp_is_typec_output(struct apple_dcp *d) { (void)d;return true; } static void complete_all(int *p) { (*p)++; } #define dcp_mode_invalidate(s) ((s)->valid=false) @@ -1285,6 +1293,7 @@ def hotplug(root): connected=0;d.typec_crtc_off=true;dcpep_cb_hotplug(&d,&connected);assert(d.rate_proof.sequence==sequence); d.typec_crtc_off=false;d.iomfb.presentation_paused=true;dcpep_cb_hotplug(&d,&connected); assert(d.rate_proof.sequence==sequence && !con.edid_invalidation_pending);d.iomfb.lock=false;probe(true); + assert(!d.typec_display_ready); dcpavserv_teardown(&av_services[0]);cleanup(); puts("PASS: actual RX HPD loss immediately denies getter/lookup before deferred work; 24 fresh-proof recovery orders need only one rate/lane callback; bootstrap/power/paused suppression preserved"); }""".replace('MAIN_PREFIX',n['main_prefix']) diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/run.py b/tools/testing/selftests/drivers/gpu/apple-dcp/run.py index 18374e2337a58..05a825197e978 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/run.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/run.py @@ -155,7 +155,9 @@ def main(): '#define iomfb_m4_get_mode(...) (-95)\n' '#define iomfb_m4_blank(...) (-95)\n') if 'dcp_fail_stop(' in unit: + # Firmware 27 fail-stops like H17P; no fixture here runs firmware 27. prefix += ('struct apple_dcp;\n' + '#define DCP_FIRMWARE_V_27_0 0x2700\n' 'static inline bool dcp_fail_stop(const struct apple_dcp *dcp);\n') unit += '\n' + block((apple / 'dcp-internal.h').read_text(), 'static inline bool dcp_fail_stop(') diff --git a/tools/testing/selftests/drivers/gpu/asahi-g15/run.py b/tools/testing/selftests/drivers/gpu/asahi-g15/run.py index 2ae5be4b57e56..b0c3105235f7b 100644 --- a/tools/testing/selftests/drivers/gpu/asahi-g15/run.py +++ b/tools/testing/selftests/drivers/gpu/asahi-g15/run.py @@ -335,7 +335,7 @@ def main(): harness += f'#[path = "{G15 / "recovery/logic.rs"}"] mod recovery;\n' harness += f'#[path = "{G15 / "power/idle.rs"}"] mod idle;\n' harness += f'#[path = "{G15 / "power/pstate.rs"}"] mod pstate;\n' - harness += f'#[path = "{G15 / "retain.rs"}"] mod retain;\n' + harness += f'#[path = "{G15.parent / "retain.rs"}"] mod retain;\n' harness += ("mod firmware_identity {\n" + f'#[path = "{G15 / "identity/matching.rs"}"] pub mod matching;\n' + f'#[path = "{G15 / "identity/lookup.rs"}"] pub mod lookup;\n' + "}\n") diff --git a/tools/testing/selftests/drivers/gpu/asahi-m4/test-command-stream.py b/tools/testing/selftests/drivers/gpu/asahi-m4/test-command-stream.py index 930b97ac24326..b94595827a791 100755 --- a/tools/testing/selftests/drivers/gpu/asahi-m4/test-command-stream.py +++ b/tools/testing/selftests/drivers/gpu/asahi-m4/test-command-stream.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: GPL-2.0-only OR MIT """Host-check production M4 Rust stream parsing with generated kernel UAPI types.""" from common import kernel_root, output_dir, rustc, flat_bindings +from abi import c_layout, differences, rust_layout import hashlib import json @@ -13,7 +14,41 @@ ROOT = kernel_root() OUT = output_dir() SOURCE = ROOT / 'drivers/gpu/drm/asahi/g16/command.rs' -BINDINGS = flat_bindings() +FLAT = flat_bindings() +# The M5 render payload and resource commands are newer than the frozen M4 +# bindings. Declare them beside the flat ones and check their layout and values +# against the kernel header, as the frozen ones are. +M5_UAPI = r''' +#[repr(C)] +#[derive(Debug, Default, Copy, Clone)] +pub struct drm_asahi_render_stage_prefixes { + pub ta_cdm_barrier: __u16, + pub reserved0: __u16, + pub reserved1: __u32, + pub ta_prior_cdm: __u64, + pub ta_vdm_barrier: __u16, + pub reserved2: __u16, + pub reserved3: __u32, + pub ta_prior_vdm: __u64, +} +#[repr(C)] +#[derive(Debug, Default, Copy, Clone)] +pub struct drm_asahi_cmd_render_m5 { + pub legacy: drm_asahi_cmd_render, + pub stage_prefixes: drm_asahi_render_stage_prefixes, +} +pub const drm_asahi_cmd_type_DRM_ASAHI_SET_RENDER_RESOURCES: drm_asahi_cmd_type = 5; +pub const drm_asahi_cmd_type_DRM_ASAHI_SET_COMPUTE_RESOURCES: drm_asahi_cmd_type = 6; +pub const drm_asahi_cmd_type_DRM_ASAHI_SET_RENDER_RESOURCES_V2: drm_asahi_cmd_type = 7; +pub const drm_asahi_render_flags_DRM_ASAHI_RENDER_STAGE_PREFIXES: drm_asahi_render_flags = 8388608; +''' +BINDINGS = OUT / 'm5-uapi.rs' +BINDINGS.write_text('include!(' + json.dumps(str(FLAT)) + ');\n' + M5_UAPI) +with tempfile.TemporaryDirectory(prefix='m4-m5-uapi-') as name: + errors = differences(c_layout(ROOT / 'include/uapi/drm/asahi_drm.h', BINDINGS, pathlib.Path(name), 'm5-layout'), + rust_layout(BINDINGS, pathlib.Path(name))) +if errors: + raise RuntimeError('Kernel UAPI ABI differs from the M5 bindings: ' + json.dumps(errors)) command = SOURCE.read_text() hardware = (SOURCE.parent / 'hw.rs').read_text() capacities = '\n'.join(re.findall(r'pub\(crate\) const (?:TPC_SIZE|TILEMAP_SIZE): u64 = .*?;', hardware)) @@ -24,7 +59,8 @@ constants = command[command.index('/// Largest command buffer.'):command.index('/// A validated render command.')] parser = command[command.index('/// One command of the stream.'):command.index('/// Checks that `[start, end)`')] g17 = (ROOT / 'drivers/gpu/drm/asahi/g17/command.rs').read_text() -g17_parser = g17[g17.index('// Commands stay inline'):g17.index('#[derive(Clone, Copy, PartialEq, Eq)]')] +# The stage-prefix offset and the render payload reader precede the parser. +g17_parser = g17[g17.index('/// Where a render command\'s stage-prefix fields start'):g17.index('#[derive(Clone, Copy, PartialEq, Eq)]')] g17_constants = '\n'.join(re.findall(r'const (?:RENDER_BASE_SIZE|COMPUTE_BASE_SIZE): usize = .*?;|const (?:RENDER_TIMESTAMP_COPIES|COMPUTE_TIMESTAMP_COPIES): Range = .*?;', g17, re.S)) reader = (ROOT / 'drivers/gpu/drm/asahi/util.rs').read_text() reader = reader[reader.index('pub(crate) unsafe trait AnyBitPattern:'):] @@ -58,6 +94,21 @@ mod g17 { use super::{uapi, AnyBitPattern, Reader, Result, EINVAL, size_of, offset_of, Range}; G17_CONSTANTS + const STAGE_PREFIXES: u32 = uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_STAGE_PREFIXES; + const FRAGMENT_BARRIERS: u32 = uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_FRAGMENT_BARRIERS; + const GFP_KERNEL: u32 = 0; + pub struct Arc(std::rc::Rc); + impl Arc { fn new(value: T, _: u32) -> Result { Ok(Self(std::rc::Rc::new(value))) } } + impl Clone for Arc { fn clone(&self) -> Self { Self(self.0.clone()) } } + impl core::ops::Deref for Arc { type Target = T; fn deref(&self) -> &T { &self.0 } } + // The M5 program lists, likewise retained as their bytes. + pub struct Programs(pub Vec, pub u8); + impl Programs { + fn parse(bytes: &[u8]) -> Result { Ok(Self(bytes.to_vec(), 1)) } + fn parse_v2(bytes: &[u8]) -> Result { Ok(Self(bytes.to_vec(), 2)) } + } + pub struct ComputePrograms(pub Vec); + impl ComputePrograms { fn parse(bytes: &[u8]) -> Result { Ok(Self(bytes.to_vec())) } } // These stand-ins only retain extension bytes. This test checks the actual // parser's framing and offsets; backend resource validation has other tests. #[derive(Clone,Copy)] @@ -86,6 +137,9 @@ } G17_PARSER } +const SET_RENDER_RESOURCES: u32 = uapi::drm_asahi_cmd_type_DRM_ASAHI_SET_RENDER_RESOURCES; +const SET_COMPUTE_RESOURCES: u32 = uapi::drm_asahi_cmd_type_DRM_ASAHI_SET_COMPUTE_RESOURCES; +const SET_RENDER_RESOURCES_V2: u32 = uapi::drm_asahi_cmd_type_DRM_ASAHI_SET_RENDER_RESOURCES_V2; fn data(value: &T) -> Vec { unsafe { core::slice::from_raw_parts(value as *const T as *const u8, size_of::()).to_vec() } } @@ -174,14 +228,16 @@ r.ray_query[0].scs_layout=0x1234; let wire=entry(CMD_RENDER,&data(&r),[NONE;2]); let parsed=g17::Parser::new(&wire).next().unwrap().unwrap(); - let g17::Payload::Render(cmd,_,ray)=parsed.payload else { panic!("wrong engine") }; + let g17::Payload::Render(cmd,prefixes,_,ray,resources)=parsed.payload else { panic!("wrong engine") }; + assert!(prefixes.is_none()&&resources.is_none()); assert_eq!([cmd.prior_vdm,cmd.prior_cdm],[17,29]); assert_eq!(u64::from_le_bytes(ray[0].0[..8].try_into().unwrap()),0x1234); let c=uapi::drm_asahi_cmd_compute {scs_layout:0x1234, prior_vdm:17,prior_cdm:29,cdm_auxiliary_allocation:8,intersection_pool:0x4000,..Default::default()}; let wire=entry(CMD_COMPUTE,&data(&c),[NONE;2]); let parsed=g17::Parser::new(&wire).next().unwrap().unwrap(); - let g17::Payload::Compute(cmd,_,intersection)=parsed.payload else { panic!("wrong engine") }; + let g17::Payload::Compute(cmd,_,intersection,resources)=parsed.payload else { panic!("wrong engine") }; + assert!(resources.is_none()); assert_eq!([cmd.prior_vdm,cmd.prior_cdm],[17,29]); assert_eq!(cmd.scs_layout,0x1234); assert_eq!(u32::from_le_bytes(intersection.0[..4].try_into().unwrap()),8); assert_eq!(u64::from_le_bytes(intersection.0[8..16].try_into().unwrap()),0x4000); @@ -198,6 +254,55 @@ } } #[test] +fn g17_stage_prefixes_follow_the_complete_render() { + let mut m5=uapi::drm_asahi_cmd_render_m5 {legacy:render(),..Default::default()}; + m5.legacy.flags=uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_STAGE_PREFIXES; m5.legacy.prior_vdm=17; + m5.stage_prefixes.ta_cdm_barrier=3; m5.stage_prefixes.ta_prior_vdm=0x1234; + let wire=entry(CMD_RENDER,&data(&m5),[NONE;2]); + let parsed=g17::Parser::new(&wire).next().unwrap().unwrap(); + let g17::Payload::Render(cmd,prefixes,..)=parsed.payload else { panic!("wrong engine") }; + let prefixes=prefixes.expect("stage prefixes"); + assert_eq!(cmd.prior_vdm,17); assert_eq!((prefixes.ta_cdm_barrier,prefixes.ta_prior_vdm),(3,0x1234)); + // The flag needs the complete struct, its reserved fields zero, and no fragment barriers. + assert!(g17::Parser::new(&entry(CMD_RENDER,&data(&m5)[..448],[NONE;2])).next().is_err()); + let mut bad=m5; bad.stage_prefixes.reserved1=1; + assert!(g17::Parser::new(&entry(CMD_RENDER,&data(&bad),[NONE;2])).next().is_err()); + let mut bad=m5; bad.legacy.flags|=FRAGMENT_BARRIERS; + assert!(g17::Parser::new(&entry(CMD_RENDER,&data(&bad),[NONE;2])).next().is_err()); + // Without the flag the prefix fields are absent or zero. + let mut plain=m5; plain.legacy.flags=0; + assert!(g17::Parser::new(&entry(CMD_RENDER,&data(&plain),[NONE;2])).next().is_err()); + plain.stage_prefixes=Default::default(); + let parsed=g17::Parser::new(&entry(CMD_RENDER,&data(&plain),[NONE;2])).next().unwrap().unwrap(); + let g17::Payload::Render(_,prefixes,..)=parsed.payload else { panic!("wrong engine") }; + assert!(prefixes.is_none()); +} +#[test] +fn g17_resource_lists_apply_to_later_commands_until_cleared() { + let mut stream=entry(SET_RENDER_RESOURCES,&[1,2,3,4],[NONE;2]); + stream.extend(entry(SET_COMPUTE_RESOURCES,&[5,6],[NONE;2])); + stream.extend(entry(CMD_RENDER,&data(&render()),[NONE;2])); + stream.extend(entry(CMD_COMPUTE,&data(&compute()),[NONE;2])); + stream.extend(entry(SET_RENDER_RESOURCES_V2,&[7],[NONE;2])); + stream.extend(entry(CMD_RENDER,&data(&render()),[NONE;2])); + stream.extend(entry(SET_RENDER_RESOURCES,&[],[NONE;2])); + stream.extend(entry(CMD_RENDER,&data(&render()),[NONE;2])); + let mut parser=g17::Parser::new(&stream); + let mut renders=Vec::new(); + while let Some(command)=parser.next().unwrap() { + match command.payload { + g17::Payload::Render(..,resources)=>renders.push(resources.map(|r|(r.0.clone(),r.1))), + g17::Payload::Compute(..,resources)=>assert_eq!(resources.map(|r|r.0.clone()),Some(vec![5,6])), + } + } + assert!(parser.finish().is_ok()); + assert_eq!(renders,[Some((vec![1,2,3,4],1)),Some((vec![7],2)),None]); + // Like the attachment commands, they carry no barriers. + for kind in [SET_RENDER_RESOURCES,SET_COMPUTE_RESOURCES,SET_RENDER_RESOURCES_V2] { + assert!(g17::Parser::new(&entry(kind,&[1],[0,NONE])).next().is_err()); + } +} +#[test] fn unsupported_flags_and_compute_fields() { for flag in [1u32,4,1<<19,FRAGMENT_BARRIERS] { let mut r=render(); r.flags=flag; @@ -310,6 +415,6 @@ 'g17_source':str(ROOT/'drivers/gpu/drm/asahi/g17/command.rs'), 'g17_sha256':hashlib.sha256(g17.encode()).hexdigest(), 'uapi_generated':str(BINDINGS),'uapi_sha256':hashlib.sha256(BINDINGS.read_bytes()).hexdigest(), - 'result':'pass','test_groups':14, + 'result':'pass','test_groups':16, 'limitations':'Links complete production encoder/firmware helper modules; compiles isolated production G16 structural parser/reader and G17 framing/prefix parser with generated UAPI structs. G17 extension validators are stand-ins; does not exercise kernel VM/scheduler or hardware.' },indent=2)+'\n') diff --git a/tools/testing/selftests/drivers/gpu/asahi-m4/test-vm-words.py b/tools/testing/selftests/drivers/gpu/asahi-m4/test-vm-words.py index 3419bca8cd181..901b1264553ea 100755 --- a/tools/testing/selftests/drivers/gpu/asahi-m4/test-vm-words.py +++ b/tools/testing/selftests/drivers/gpu/asahi-m4/test-vm-words.py @@ -81,7 +81,7 @@ def body(source,marker): use std::{cell::Cell, rc::Rc, ops::{Range,Deref}};use std::sync::atomic::{AtomicBool,Ordering}; macro_rules! dev_err { ($($arg:tt)*) => {}; } type Result = std::result::Result; -const EOVERFLOW:u32=75; +const EOVERFLOW:u32=75;const EBUSY:u32=16; #[derive(Clone,Copy)]struct Prot; struct KernelMappingInner{mapped_size:usize} mod mm { @@ -95,16 +95,18 @@ def body(source,marker): impl Uat{ fn lock_m4_upper(self:&Rc,upper:bool)->Result{assert!(upper);assert!(!self.held.replace(true),"recursive shared-UAT lock");Ok(Lease(self.clone()))} } -struct Table{uat:Rc,rolled_back:Cell} +struct Table{uat:Rc,rolled_back:Cell,busy:bool} impl Table{ fn prepare_map(&mut self,_r:Range)->Result{assert!(!self.uat.held.get(),"allocation under PPL");Ok(())} fn discard_partial_map(&mut self,_r:Range)->Result{assert!(self.uat.held.get(),"rollback outside PPL");self.rolled_back.set(true);Ok(())} + // Another mapping's live leaves in the range refuse the map before any store. + fn ensure_unmapped(&mut self,_r:Range)->Result{assert!(self.uat.held.get(),"live-leaf check outside PPL");if self.busy{Err(EBUSY)}else{Ok(())}} } -struct Owner{uat_inner:Rc,page_table:Table,is_kernel:bool,fail:bool,invalidations:Cell} +struct Owner{uat_inner:Rc,page_table:Table,is_kernel:bool,fail:bool,invalidations:Cell,stores:Cell} impl Owner{ fn mapping_mutation(&mut self){} fn map_node_prepared(&mut self,_n:&mm::Node<(),KernelMappingInner>,_p:Prot)->Result{ - assert!(self.uat_inner.held.get(),"live leaf stores outside PPL");if self.fail{Err(5)}else{Ok(())} + assert!(self.uat_inner.held.get(),"live leaf stores outside PPL");self.stores.set(self.stores.get()+1);if self.fail{Err(5)}else{Ok(())} } fn tlbi_contexts_naming_root(&self,_va:u64,_size:usize){assert!(!self.uat_inner.held.get(),"naming-root lookup recursively locks UatShared");self.invalidations.set(self.invalidations.get()+1);} fn finish_failed_map(&self,va:u64,size:usize){self.tlbi_contexts_naming_root(va,size);} @@ -113,12 +115,20 @@ def body(source,marker): #[test]fn success_and_rollback_release_ppl_before_root_lookup(){ for fail in [false,true]{ let uat=Rc::new(Uat{held:Cell::new(false),m4_adopted:true,failed:AtomicBool::new(false)}); - let mut owner=Owner{uat_inner:uat.clone(),page_table:Table{uat:uat.clone(),rolled_back:Cell::new(false)},is_kernel:true,fail,invalidations:Cell::new(0)}; + let mut owner=Owner{uat_inner:uat.clone(),page_table:Table{uat:uat.clone(),rolled_back:Cell::new(false),busy:false},is_kernel:true,fail,invalidations:Cell::new(0),stores:Cell::new(0)}; let node=mm::Node{start:0xfffffc2000000000,inner:KernelMappingInner{mapped_size:0x4000},marker:core::marker::PhantomData}; assert_eq!(owner.map_node(&node,Prot),if fail{Err(5)}else{Ok(())}); assert!(!uat.held.get());assert_eq!(owner.page_table.rolled_back.get(),fail);assert_eq!(owner.invalidations.get(),1); } } +#[test]fn a_live_range_is_refused_before_any_store_and_releases_ppl(){ + let uat=Rc::new(Uat{held:Cell::new(false),m4_adopted:true,failed:AtomicBool::new(false)}); + let mut owner=Owner{uat_inner:uat.clone(),page_table:Table{uat:uat.clone(),rolled_back:Cell::new(false),busy:true},is_kernel:true,fail:false,invalidations:Cell::new(0),stores:Cell::new(0)}; + let node=mm::Node{start:0xfffffc2000000000,inner:KernelMappingInner{mapped_size:0x4000},marker:core::marker::PhantomData}; + assert_eq!(owner.map_node(&node,Prot),Err(EBUSY)); + assert!(!uat.held.get());assert_eq!(owner.stores.get(),0);assert!(!owner.page_table.rolled_back.get()); + assert_eq!(owner.invalidations.get(),0);assert!(!uat.failed.load(Ordering::Acquire)); +} '''.replace('__MAP_TRANSACTION__',map_transaction) lock_unit=OUT/'map-node-lock.rs';lock_unit.write_text(lock_source) subprocess.run(rustc()+['--edition=2021','--test',str(lock_unit),'-o',str(OUT/'map-node-lock')],check=True)